diff --git a/DOE_SYMBOLIC_PR_NOTES.md b/DOE_SYMBOLIC_PR_NOTES.md new file mode 100644 index 00000000000..0e0020f247f --- /dev/null +++ b/DOE_SYMBOLIC_PR_NOTES.md @@ -0,0 +1,202 @@ +# Symbolic DoE PR Notes + +This note is intended to help reviewers understand the scope of the symbolic +Pyomo.DoE pull request, the mathematical background for the main changes, and +how those ideas map onto the implementation. + +## Overview + +This PR adds symbolic-gradient support to `pyomo.contrib.doe` on top of the +current DoE implementation while preserving the newer objective and GreyBox +structure already present on `main`. + +At a high level, the PR: + +- adds symbolic / `pynumero` gradient support alongside the existing + finite-difference workflow +- adds `ExperimentGradients` to organize the derivative information used by DoE +- keeps the newer DoE objective and GreyBox structure from current `main` +- adds a lightweight public `PolynomialExperiment` example for symbolic testing +- broadens test coverage for symbolic/automatic derivative consistency and DoE + regression behavior + +The PR does **not** change the underlying Fisher information matrix mathematics. +The main change is how the derivative information needed for those calculations +can be assembled. + +## Mathematical Background + +For a model with outputs `y` and unknown parameters `theta`, DoE needs the +output sensitivity matrix + +```math +Q = \frac{\partial y}{\partial \theta} +``` + +to build the Fisher information matrix + +```math +\mathrm{FIM} = Q^T \Sigma^{-1} Q +``` + +where `Sigma` is the measurement-error covariance matrix. + +For models defined implicitly by equations + +```math +F(x, u, \theta) = 0 +``` + +the state sensitivities follow from differentiating the implicit system: + +```math +\frac{\partial F}{\partial x}\frac{\partial x}{\partial \theta} ++ +\frac{\partial F}{\partial \theta} += 0 +``` + +so + +```math +\frac{\partial x}{\partial \theta} += +-\left(\frac{\partial F}{\partial x}\right)^{-1} +\frac{\partial F}{\partial \theta} +``` + +Those state sensitivities are then propagated to the experiment outputs to form +`Q`, and then to the FIM. + +The symbolic path in this PR changes how those Jacobian terms are assembled; it +does not change the equations above. + +## Implementation Mapping + +### `gradient_method` + +The symbolic behavior in this PR is selected through `gradient_method`. + +- finite-difference setup still uses `fd_formula` +- symbolic / `pynumero` behavior is selected by `gradient_method` +- `fd_formula="central"` remains the shared setup convention in many tests even + when `gradient_method="pynumero"` is used + +This means the symbolic path is not activated by changing the finite-difference +formula. Instead, the finite-difference configuration stays available while the +derivative backend is chosen independently. + +### `ExperimentGradients` + +`ExperimentGradients` is responsible for organizing the derivative information +used to build sensitivity matrices for DoE. + +The important implementation detail in this PR is that the symbolic and +automatic derivative structures are prepared through a unified setup path. This +lets tests compare symbolic and automatic Jacobian entries directly while still +using the same overall experiment structure. + +In other words, the PR moves the code toward: + +- one shared setup path for Jacobian bookkeeping +- backend-specific derivative population within that structure + +rather than maintaining more separate symbolic and automatic setup logic. + +## Test Strategy + +The test updates in this PR were aimed at checking both the mathematical path +and the practical DoE integration path. + +### Polynomial + +The polynomial example is the lightweight symbolic reference problem. + +It is used for: + +- exact gradient checks against hand-derived values +- symbolic vs automatic Jacobian consistency checks +- public example coverage +- generic 2D factorial / plotting coverage where a small two-design-variable + example is helpful + +Because the polynomial example has one output and four parameters, some FIM +regression tests use an identity prior to avoid rank-deficient raw FIMs when the +test purpose is metric regression rather than singular-matrix behavior. + +### Rooney-Biegler + +Rooney-Biegler is used for most of the general-purpose solve and regression +coverage because it is lightweight and still exercises the full DoE flow. + +It is used for: + +- symbolic `run_doe()` regression tests +- symbolic objective-matrix consistency checks +- bad-model / error-path coverage +- perturbed-point Jacobian agreement checks in the non-reactor replacements +- GreyBox helper and solve-path coverage + +Several Rooney-Biegler determinant tests use a prior FIM. This is not meant to +change the underlying mathematics; it is there to keep the determinant-based +solve path well-conditioned for the small Rooney-Biegler problem. + +### GreyBox + +The non-solve GreyBox helper tests now use the Rooney-Biegler path as well. + +Those tests were updated to match a two-parameter FIM instead of the older +reactor-specific four-parameter setup. In particular: + +- the test FIM used for GreyBox helper checks is now `2 x 2` +- the reduced-Hessian finite-difference helper was generalized to use the + current FIM dimension instead of assuming four parameters +- the build checks compare against the actual FIM carried by the GreyBox object + rather than a hard-coded reactor-specific reconstruction + +The solve-path checks still use the `cyipopt` GreyBox route and therefore remain +subject to the MA57/HSL runtime availability for that path. + +## Notes And Caveats + +### Reactor initialization nuance + +One important nuance that came up during review is that the reactor model +required more care for generalized symbolic/automatic correctness checks. + +In particular: + +- the raw reactor labeled model is not always the cleanest reduced test vehicle + for output-sensitivity checks +- some reactor-oriented checks were therefore replaced with lighter + Rooney-Biegler or polynomial coverage where the test purpose was generic + +### MA57 / HSL on the GreyBox `cyipopt` path + +The GreyBox `cyipopt` path is distinct from simply having a working standalone +IPOPT executable. In local debugging, the relevant failure mode was that +`cyipopt` could be present while the MA57/HSL runtime needed on that path was +not available. The test skip message and code comment call that out explicitly +so it is not confused with generic IPOPT availability. + +## Validation Snapshot + +The final focused DoE test bundle used during this review pass was: + +```bash +python -m pytest -q \ + pyomo/contrib/doe/tests/test_utils.py \ + pyomo/contrib/doe/tests/test_doe_solve.py \ + pyomo/contrib/doe/tests/test_doe_build.py \ + pyomo/contrib/doe/tests/test_doe_errors.py \ + pyomo/contrib/doe/tests/test_greybox.py +``` + +with local result: + +```text +130 passed, 4 skipped, 5 warnings in 35.67s +``` + +The remaining warnings are the expected non-interactive matplotlib `Agg` +warnings from tests that call `draw_factorial_figure()`. diff --git a/doc/OnlineDocs/explanation/analysis/doe/abstraction.rst b/doc/OnlineDocs/explanation/analysis/doe/abstraction.rst new file mode 100644 index 00000000000..4aabe3a9f3a --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/doe/abstraction.rst @@ -0,0 +1,8 @@ +.. _abstractexp: + +Experiment Abstraction +====================== + +.. note:: + + Detailed descriptions and example code for experiment abstraction in Pyomo.DoE will be added in a future update. diff --git a/doc/OnlineDocs/explanation/analysis/doe/doe.rst b/doc/OnlineDocs/explanation/analysis/doe/doe.rst index 77fc8b1a08a..7cefb99e418 100644 --- a/doc/OnlineDocs/explanation/analysis/doe/doe.rst +++ b/doc/OnlineDocs/explanation/analysis/doe/doe.rst @@ -21,363 +21,15 @@ If you use Pyomo.DoE, please cite: "Pyomo.DOE: An open‐source package for model‐based design of experiments in Python." AIChE Journal 68.12 (2022): e17813. `https://doi.org/10.1002/aic.17813` -Methodology Overview ---------------------- - -Model-based Design of Experiments (MBDoE) is a technique to maximize the information -gain from experiments by directly using science-based models with physically meaningful -parameters. It is one key component in the model calibration and uncertainty -quantification workflow shown below: - -.. figure:: pyomo_workflow_new.png - :align: center - :scale: 90 % - -The parameter estimation, uncertainty analysis, and MBDoE are -combined into an iterative framework to select, refine, and calibrate science-based -mathematical models with quantified uncertainty. Currently, Pyomo.DoE focuses on -increasing parameter precision. - -Pyomo.DoE provides the exploratory analysis and MBDoE capabilities to the -Pyomo ecosystem. The user provides one Pyomo model, a set of parameter nominal values, -the allowable design spaces for design variables, and the assumed observation error model. -During exploratory analysis, Pyomo.DoE checks if the model parameters can be -inferred from the postulated measurements or preliminary data. -MBDoE then recommends optimized experimental conditions for collecting more data. -Parameter estimation packages such as :ref:`Parmest ` can perform -parameter estimation using the available data to infer values for parameters, -and facilitate an uncertainty analysis to approximate the parameter covariance matrix. -If the parameter uncertainties are sufficiently small, the workflow terminates -and returns the final model with quantified parametric uncertainty. -If not, MBDoE recommends optimized experimental conditions to generate new data -that will maximize information gain and eventually reduce parameter uncertainty. - -Below is an overview of the type of optimization models Pyomo.DoE can accommodate: - -* Pyomo.DoE is suitable for optimization models of **continuous** variables -* Pyomo.DoE can handle **equality constraints** defining state variables -* Pyomo.DoE supports (Partial) Differential-Algebraic Equations (PDAE) models via :ref:`Pyomo.DAE ` -* Pyomo.DoE also supports models with only algebraic equations - -The general form of a DAE problem that can be passed into Pyomo.DoE is shown below: - -.. math:: - :nowrap: - - \[\begin{array}{l} - \dot{\mathbf{x}}(t) = \mathbf{f}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}}, \boldsymbol{\theta}) \\ - \mathbf{g}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\theta})=\mathbf{0} \\ - \mathbf{y} =\mathbf{h}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\theta}) \\ - \mathbf{f}^{\mathbf{0}}\left(\dot{\mathbf{x}}\left(t_{0}\right), \mathbf{x}\left(t_{0}\right), \mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\theta}\right)=\mathbf{0} \\ - \mathbf{g}^{\mathbf{0}}\left( \mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\theta}\right)=\mathbf{0}\\ - \mathbf{y}^{\mathbf{0}}\left(t_{0}\right)=\mathbf{h}\left(\mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\theta}\right) - \end{array}\] - -where: - -* :math:`\boldsymbol{\theta} \in \mathbb{R}^{N_p}` are unknown model parameters. -* :math:`\mathbf{x} \subseteq \mathcal{X}` are dynamic state variables which characterize trajectory of the system, :math:`\mathcal{X} \in \mathbb{R}^{N_x \times N_t}`. -* :math:`\mathbf{z} \subseteq \mathcal{Z}` are algebraic state variables, :math:`\mathcal{Z} \in \mathbb{R}^{N_z \times N_t}`. -* :math:`\mathbf{u} \subseteq \mathcal{U}` are time-varying decision variables, :math:`\mathcal{U} \in \mathbb{R}^{N_u \times N_t}`. -* :math:`\overline{\mathbf{w}} \in \mathbb{R}^{N_w}` are time-invariant decision variables. -* :math:`\mathbf{y} \subseteq \mathcal{Y}` are measurement response variables, :math:`\mathcal{Y} \in \mathbb{R}^{N_r \times N_t}`. -* :math:`\mathbf{f}(\cdot)` are differential equations. -* :math:`\mathbf{g}(\cdot)` are algebraic equations. -* :math:`\mathbf{h}(\cdot)` are measurement functions. -* :math:`\mathbf{t} \in \mathbb{R}^{N_t \times 1}` is a union of all time sets. - -.. note:: - Parameters and design variables should be defined as Pyomo ``Var`` components - when building the model using the ``Experiment`` class so that users can use both - ``Parmest`` and ``Pyomo.DoE`` seamlessly. - -Based on the above notation, the form of the MBDoE problem addressed in Pyomo.DoE is shown below: - -.. math:: - :nowrap: - - \begin{equation} - \begin{aligned} - \underset{\boldsymbol{\varphi}}{\max} \quad & \Psi (\mathbf{M}(\boldsymbol{\hat{\theta}}, \boldsymbol{\varphi})) \\ - \text{s.t.} \quad & \mathbf{M}(\boldsymbol{\hat{\theta}}, \boldsymbol{\varphi}) = \sum_r^{N_r} \sum_{r'}^{N_r} \tilde{\sigma}_{(r,r')}\mathbf{Q}_r^\mathbf{T} \mathbf{Q}_{r'} + \mathbf{M}_0 \\ - & \dot{\mathbf{x}}(t) = \mathbf{f}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}}) \\ - & \mathbf{g}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\hat{\theta}})=\mathbf{0} \\ - & \mathbf{y} =\mathbf{h}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\hat{\theta}}) \\ - & \mathbf{f}^{\mathbf{0}}\left(\dot{\mathbf{x}}\left(t_{0}\right), \mathbf{x}\left(t_{0}\right), \mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}})\right)=\mathbf{0} \\ - & \mathbf{g}^{\mathbf{0}}\left( \mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}}\right)=\mathbf{0}\\ - &\mathbf{y}^{\mathbf{0}}\left(t_{0}\right)=\mathbf{h}\left(\mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}}\right) - \end{aligned} - \end{equation} - -where: - -* :math:`\boldsymbol{\varphi}` are design variables, which are manipulated to maximize the information content of experiments. It should consist of one or more of :math:`\mathbf{u}(t), \mathbf{y}^{\mathbf{0}}({t_0}),\overline{\mathbf{w}}`. With a proper model formulation, the timepoints for control or measurements :math:`\mathbf{t}` can also be degrees of freedom. -* :math:`\mathbf{M}` is the Fisher information matrix (FIM), approximated as the inverse of the covariance matrix of parameter estimates :math:`\boldsymbol{\hat{\theta}}`. A large FIM indicates more information contained in the experiment for parameter estimation. -* :math:`\mathbf{Q}` is the dynamic sensitivity matrix, containing the partial derivatives of :math:`\mathbf{y}` with respect to :math:`\boldsymbol{\theta}`. -* :math:`\Psi` is the scalar design criteria to measure the information content in the FIM. -* :math:`\mathbf{M}_0` is the sum of all the FIMs from previous experiments. - -Pyomo.DoE provides five design criteria :math:`\Psi` to measure the information in the FIM. -The covariance matrix of parameter estimates is approximated as the inverse of the FIM, -i.e., :math:`\mathbf{V} \approx \mathbf{M}^{-1}`. -We can use the FIM or the covariance matrix to define the design criteria. - -.. list-table:: Pyomo.DoE design criteria - :header-rows: 1 - :class: tight-table - - * - Design criterion - - Computation - - Geometrical meaning - * - A-optimality - - :math:`\text{trace}(\mathbf{V}) = \text{trace}(\mathbf{M}^{-1})` - - Minimizing this is equivalent to minimizing the enclosing box of the confidence ellipse - * - Pseudo A-optimality - - :math:`\text{trace}(\mathbf{M})` - - Maximizing this is equivalent to maximizing the dimensions of the enclosing box of the Fisher Information Matrix - * - D-optimality - - :math:`\det(\mathbf{M}) = 1/\det(\mathbf{V})` - - Maximizing this is equivalent to minimizing confidence-ellipsoid volume - * - E-optimality - - :math:`\lambda_{\min}(\mathbf{M}) = 1/\lambda_{\max}(\mathbf{V})` - - Maximizing this is equivalent to minimizing the longest axis of the confidence ellipse - * - Modified E-optimality - - :math:`\text{cond}(\mathbf{M}) = \text{cond}(\mathbf{V})` - - Minimizing this is equivalent to minimizing the ratio of the longest axis to the shortest axis of the confidence ellipse - -.. note:: - A confidence ellipse is a geometric representation of the uncertainty in parameter - estimates. It is derived from the covariance matrix :math:`\mathbf{V}`. - -In order to solve problems of the above, Pyomo.DoE implements the 2-stage stochastic program. Please see Wang and Dowling (2022) for details. - -Pyomo.DoE Required Inputs -------------------------- -To use Pyomo.DoE, a user must implement a subclass of the :ref:`Parmest ` ``Experiment`` class. -The subclass must have a ``get_labeled_model`` method which returns a Pyomo `ConcreteModel` -containing four Pyomo ``Suffix`` components identifying the parts of the model used in -MBDoE analysis. This is in line with the convention used in the parameter estimation tool, -:ref:`Parmest `. The four Pyomo ``Suffix`` components are: - -* ``experiment_inputs`` - The experimental design decisions -* ``experiment_outputs`` - The values measured during the experiment -* ``measurement_error`` - The error associated with individual values measured during the experiment. It is passed as a standard deviation or square root of the diagonal elements of the observation error covariance matrix. Pyomo.DoE currently assumes that the observation errors are Gaussain and independent both in time and across measurements. -* ``unknown_parameters`` - Those parameters in the model that are estimated using the measured values during the experiment - -An example of the subclassed ``Experiment`` object that builds and labels the model is shown in the next few sections. - -Pyomo.DoE Usage Example ------------------------ - -We illustrate the use of Pyomo.DoE using a reaction kinetics example (Wang and Dowling, 2022). - -.. math:: - :nowrap: - - \begin{equation} - A \xrightarrow{k_1} B \xrightarrow{k_2} C - \end{equation} - - -The Arrhenius equations model the temperature dependence of the reaction rate coefficients :math:`k_1` and :math:`k_2`. Assuming a first-order reaction mechanism gives the reaction rate model shown below. Further, we assume only species A is fed to the reactor. - -.. math:: - :nowrap: - - \begin{equation} - \begin{aligned} - k_1 & = A_1 e^{-\frac{E_1}{RT}} \\ - k_2 & = A_2 e^{-\frac{E_2}{RT}} \\ - \frac{d{C_A}}{dt} & = -k_1{C_A} \\ - \frac{d{C_B}}{dt} & = k_1{C_A} - k_2{C_B} \\ - C_{A0}& = C_A + C_B + C_C \\ - C_B(t_0) & = 0 \\ - C_C(t_0) & = 0 \\ - \end{aligned} - \end{equation} - - - -:math:`C_A(t), C_B(t), C_C(t)` are the time-varying concentrations of the species A, B, C, respectively. -:math:`k_1, k_2` are the rate constants for the two chemical reactions using an Arrhenius equation with activation energies :math:`E_1, E_2` and pre-exponential factors :math:`A_1, A_2`. -The goal of MBDoE is to optimize the experiment design variables :math:`\boldsymbol{\varphi} = (C_{A0}, T(t))`, where :math:`C_{A0},T(t)` are the initial concentration of species A and the time-varying reactor temperature, to maximize the precision of unknown model parameters :math:`\boldsymbol{\theta} = (A_1, E_1, A_2, E_2)` by measuring :math:`\mathbf{y}(t)=(C_A(t), C_B(t), C_C(t))`. -The observation errors are assumed to be independent both in time and across measurements with a constant standard deviation of 1 M for each species. - - -Step 0: Import Pyomo and the Pyomo.DoE module and create an ``Experiment`` class -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. note:: - - This example uses the data file ``result.json``, located in the Pyomo repository at: - ``pyomo/contrib/doe/examples/result.json``, which contains the nominal parameter - values, and measurements for the reaction kinetics experiment. - - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py - :start-after: # === Required imports === - :end-before: # ======================== - -Subclass the :ref:`Parmest ` ``Experiment`` class to define the reaction -kinetics experiment and build the Pyomo ConcreteModel. - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py - :start-after: ======================== - :end-before: End constructor definition - -Step 1: Define the Pyomo process model -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The process model for the reaction kinetics problem is shown below. Here, we build -the model without any data or discretization. - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py - :start-after: Create flexible model without data - :end-before: End equation definition - -Step 2: Finalize the Pyomo process model -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Here, we add data to the model and finalize the discretization using a new method to -the class. This step is required before the model can be labeled. - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py - :start-after: End equation definition - :end-before: End model finalization - -Step 3: Label the information needed for DoE analysis -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -We label the four important groups as Pyomo Suffix components as mentioned before by -adding a ``label_experiment`` method. This method is required by Pyomo.DoE to identify -the design variables (experimental inputs), measurements, measurement errors, and -unknown parameters in the model. - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py - :start-after: End model finalization - :end-before: End model labeling - -Step 4: Implement the ``get_labeled_model`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This method utilizes the previous 3 steps and is used by `Pyomo.DoE` to build the model -to perform optimal experimental design. - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py - :start-after: End constructor definition - :end-before: Create flexible model without data - -Step 5: Exploratory analysis (Enumeration) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -After creating the subclass of the ``Experiment`` class, exploratory analysis is -suggested to enumerate the design space to check if the problem is identifiable, -i.e., ensure that D-, E-optimality metrics are not small numbers near zero, and -Modified E-optimality is not a big number. -Additionally, it helps to initialize the model for the optimal experimental design step. - -Pyomo.DoE can perform exploratory sensitivity analysis with the ``compute_FIM_full_factorial`` method. -The ``compute_FIM_full_factorial`` method generates a grid over the design space as specified by the user. -Each grid point represents an MBDoE problem solved using the ``compute_FIM`` method. -In this way, sensitivity of the FIM over the design space can be evaluated. -Pyomo.DoE supports plotting the results from the ``compute_FIM_full_factorial`` method -with the ``draw_factorial_figure`` method. - -The following code defines the ``run_reactor_doe`` function. This function encapsulates -the workflow for both sensitivity analysis (Step 5) and optimal design (Step 6). - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_example.py - :language: python - :start-after: # === Required imports === - :end-before: if __name__ == "__main__": - -After defining the function, we will call it to perform the exploratory analysis and -the optimal experimental design. - -.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_example.py - :language: python - :start-after: if __name__ == "__main__": - :dedent: 4 - -A design exploration for the initial concentration and temperature as experimental -design variables with 9 values for each, produces the the five figures for -five optimality criteria using the ``compute_FIM_full_factorial`` and -``draw_factorial_figure`` methods as shown below: - -|plot1| |plot2| - -|plot3| |plot4| - -|plot5| - -.. |plot1| image:: example_reactor_compute_FIM_D_opt.png - :width: 48 % - -.. |plot2| image:: example_reactor_compute_FIM_A_opt.png - :width: 48 % - -.. |plot3| image:: example_reactor_compute_FIM_pseudo_A_opt.png - :width: 48 % - -.. |plot4| image:: example_reactor_compute_FIM_E_opt.png - :width: 48 % - -.. |plot5| image:: example_reactor_compute_FIM_ME_opt.png - :width: 48 % - -The heatmaps show the values of the objective functions, a.k.a. the -experimental information content, in the design space. Horizontal -and vertical axes are the two experimental design variables, while -the color of each grid shows the experimental information content. -For example, the D-optimality (upper left subplot) heatmap figure shows that the -most informative region is around :math:`C_{A0}=5.0` M, :math:`T=500.0` K with -a :math:`\log_{10}` determinant of FIM being around 19, -while the least informative region is around :math:`C_{A0}=1.0` M, :math:`T=300.0` K, -with a :math:`\log_{10}` determinant of FIM being around -5. For D-, Pseudo A-, and -E-optimality we want to maximize the objective function, while for A- and Modified -E-optimality we want to minimize the objective function. - -In this sensitivity analysis plot (heatmap), we only varied the initial -concentration and the initial temperature, while the temperature at other time -points is fixed at 300 K. - -.. math:: - :nowrap: - - \[ - T(t) = \begin{cases} - T_0, & t \le 0.125 \\ - 300\ \text{K}, & t > 0.125 - \end{cases} - \] - -If :math:`T_0 = 300\ \text{K}`, the reaction is conducted under strictly isothermal -conditions. Because the temperature is constant, the sensitivities of the species -concentrations with respect to the Arrhenius parameters (:math:`A_i` and :math:`E_i`) -become linearly dependent. This high correlation means the effects of the -pre-exponential factor and the activation energy cannot be uniquely distinguished -from the measurements. Consequently, the Fisher Information Matrix (FIM) becomes -ill-conditioned, resulting in a near-zero determinant and a very large condition number. - -To break this correlation and make the parameters identifiable, introducing a time- -varying temperature profile (for example, a temperature step or a ramp) is required. -As shown in the heatmap, when the initial temperature :math:`T_0` differs from the -subsequent 300 K baseline, such a temperature change breaks the linear dependence, -yielding a well-conditioned FIM and identifiable parameters. - - - -Step 6: Performing an optimal experimental design -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In Step 5, we defined the ``run_reactor_doe`` function. This function constructs -the DoE object and performs the exploratory sensitivity analysis. The way the function -is defined, it also proceeds immediately to the optimal experimental design step -(applying ``run_doe`` on the ``DesignOfExperiments`` object). -We can initialize the model with the result we obtained from the exploratory -analysis (optimal point from the heatmaps) to help the optimal design step to speed -up convergence. However, implementation of this initialization is not shown here. - -After applying ``run_doe`` on the ``DesignOfExperiments`` object, -the optimal design is an initial concentration of 5.0 mol/L and -an initial temperature of 494 K with all other temperatures being 300 K. -The corresponding :math:`\log_{10}` determinant of the FIM is 19.32. +Index of Pyomo.DoE documentation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. toctree:: + :maxdepth: 2 + + overview.rst + guide.rst + abstraction.rst + objective.rst + multexp.rst + uncertainty.rst \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/doe/guide.rst b/doc/OnlineDocs/explanation/analysis/doe/guide.rst new file mode 100644 index 00000000000..cb14c08d6e0 --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/doe/guide.rst @@ -0,0 +1,230 @@ +.. _startguide: + +Quick Start Guide +================= + +To use Pyomo.DoE, a user must implement a subclass of the :ref:`Parmest ` ``Experiment`` class. +The subclass must have a ``get_labeled_model`` method which returns a Pyomo `ConcreteModel` +containing four Pyomo ``Suffix`` components identifying the parts of the model used in +MBDoE analysis. This is in line with the convention used in the parameter estimation tool, +:ref:`Parmest `. The four Pyomo ``Suffix`` components are: + +* ``experiment_inputs`` - The experimental design decisions +* ``experiment_outputs`` - The values measured during the experiment +* ``measurement_error`` - The error associated with individual values measured during the experiment. It is passed as a standard deviation or square root of the diagonal elements of the observation error covariance matrix. Pyomo.DoE currently assumes that the observation errors are Gaussain and independent both in time and across measurements. +* ``unknown_parameters`` - Those parameters in the model that are estimated using the measured values during the experiment + +An example of the subclassed ``Experiment`` object that builds and labels the model is shown in the next few sections. + +This guide illustrates the use of Pyomo.DoE using a reaction kinetics example (Wang and Dowling, 2022). + +.. math:: + :nowrap: + + \begin{equation} + A \xrightarrow{k_1} B \xrightarrow{k_2} C + \end{equation} + + +The Arrhenius equations model the temperature dependence of the reaction rate coefficients :math:`k_1` and :math:`k_2`. Assuming a first-order reaction mechanism gives the reaction rate model shown below. Further, we assume only species A is fed to the reactor. + +.. math:: + :nowrap: + + \begin{equation} + \begin{aligned} + k_1 & = A_1 e^{-\frac{E_1}{RT}} \\ + k_2 & = A_2 e^{-\frac{E_2}{RT}} \\ + \frac{d{C_A}}{dt} & = -k_1{C_A} \\ + \frac{d{C_B}}{dt} & = k_1{C_A} - k_2{C_B} \\ + C_{A0}& = C_A + C_B + C_C \\ + C_B(t_0) & = 0 \\ + C_C(t_0) & = 0 \\ + \end{aligned} + \end{equation} + + + +:math:`C_A(t), C_B(t), C_C(t)` are the time-varying concentrations of the species A, B, C, respectively. +:math:`k_1, k_2` are the rate constants for the two chemical reactions using an Arrhenius equation with activation energies :math:`E_1, E_2` and pre-exponential factors :math:`A_1, A_2`. +The goal of MBDoE is to optimize the experiment design variables :math:`\boldsymbol{\varphi} = (C_{A0}, T(t))`, where :math:`C_{A0},T(t)` are the initial concentration of species A and the time-varying reactor temperature, to maximize the precision of unknown model parameters :math:`\boldsymbol{\theta} = (A_1, E_1, A_2, E_2)` by measuring :math:`\mathbf{y}(t)=(C_A(t), C_B(t), C_C(t))`. +The observation errors are assumed to be independent both in time and across measurements with a constant standard deviation of 1 M for each species. + + +Step 0: Import Pyomo and the Pyomo.DoE module and create an ``Experiment`` class +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: + + This example uses the data file ``result.json``, located in the Pyomo repository at: + ``pyomo/contrib/doe/examples/result.json``, which contains the nominal parameter + values, and measurements for the reaction kinetics experiment. + + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py + :start-after: # === Required imports === + :end-before: # ======================== + +Subclass the :ref:`Parmest ` ``Experiment`` class to define the reaction +kinetics experiment and build the Pyomo ConcreteModel. + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py + :start-after: ======================== + :end-before: End constructor definition + +Step 1: Define the Pyomo process model +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The process model for the reaction kinetics problem is shown below. Here, we build +the model without any data or discretization. + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py + :start-after: Create flexible model without data + :end-before: End equation definition + +Step 2: Finalize the Pyomo process model +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Here, we add data to the model and finalize the discretization using a new method to +the class. This step is required before the model can be labeled. + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py + :start-after: End equation definition + :end-before: End model finalization + +Step 3: Label the information needed for DoE analysis +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We label the four important groups as Pyomo Suffix components as mentioned before by +adding a ``label_experiment`` method. This method is required by Pyomo.DoE to identify +the design variables (experimental inputs), measurements, measurement errors, and +unknown parameters in the model. + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py + :start-after: End model finalization + :end-before: End model labeling + +Step 4: Implement the ``get_labeled_model`` method +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This method utilizes the previous 3 steps and is used by `Pyomo.DoE` to build the model +to perform optimal experimental design. + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_experiment.py + :start-after: End constructor definition + :end-before: Create flexible model without data + +Step 5: Exploratory analysis (Enumeration) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +After creating the subclass of the ``Experiment`` class, exploratory analysis is +suggested to enumerate the design space to check if the problem is identifiable, +i.e., ensure that D-, E-optimality metrics are not small numbers near zero, and +Modified E-optimality is not a big number. +Additionally, it helps to initialize the model for the optimal experimental design step. + +Pyomo.DoE can perform exploratory sensitivity analysis with the ``compute_FIM_full_factorial`` method. +The ``compute_FIM_full_factorial`` method generates a grid over the design space as specified by the user. +Each grid point represents an MBDoE problem solved using the ``compute_FIM`` method. +In this way, sensitivity of the FIM over the design space can be evaluated. +Pyomo.DoE supports plotting the results from the ``compute_FIM_full_factorial`` method +with the ``draw_factorial_figure`` method. + +The following code defines the ``run_reactor_doe`` function. This function encapsulates +the workflow for both sensitivity analysis (Step 5) and optimal design (Step 6). + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_example.py + :language: python + :start-after: # === Required imports === + :end-before: if __name__ == "__main__": + +After defining the function, we will call it to perform the exploratory analysis and +the optimal experimental design. + +.. literalinclude:: /../../pyomo/contrib/doe/examples/reactor_example.py + :language: python + :start-after: if __name__ == "__main__": + :dedent: 4 + +A design exploration for the initial concentration and temperature as experimental +design variables with 9 values for each, produces the the five figures for +five optimality criteria using the ``compute_FIM_full_factorial`` and +``draw_factorial_figure`` methods as shown below: + +|plot1| |plot2| + +|plot3| |plot4| + +|plot5| + +.. |plot1| image:: example_reactor_compute_FIM_D_opt.png + :width: 48 % + +.. |plot2| image:: example_reactor_compute_FIM_A_opt.png + :width: 48 % + +.. |plot3| image:: example_reactor_compute_FIM_pseudo_A_opt.png + :width: 48 % + +.. |plot4| image:: example_reactor_compute_FIM_E_opt.png + :width: 48 % + +.. |plot5| image:: example_reactor_compute_FIM_ME_opt.png + :width: 48 % + +The heatmaps show the values of the objective functions, a.k.a. the +experimental information content, in the design space. Horizontal +and vertical axes are the two experimental design variables, while +the color of each grid shows the experimental information content. +For example, the D-optimality (upper left subplot) heatmap figure shows that the +most informative region is around :math:`C_{A0}=5.0` M, :math:`T=500.0` K with +a :math:`\log_{10}` determinant of FIM being around 19, +while the least informative region is around :math:`C_{A0}=1.0` M, :math:`T=300.0` K, +with a :math:`\log_{10}` determinant of FIM being around -5. For D-, Pseudo A-, and +E-optimality we want to maximize the objective function, while for A- and Modified +E-optimality we want to minimize the objective function. + +In this sensitivity analysis plot (heatmap), we only varied the initial +concentration and the initial temperature, while the temperature at other time +points is fixed at 300 K. + +.. math:: + :nowrap: + + \[ + T(t) = \begin{cases} + T_0, & t \le 0.125 \\ + 300\ \text{K}, & t > 0.125 + \end{cases} + \] + +If :math:`T_0 = 300\ \text{K}`, the reaction is conducted under strictly isothermal +conditions. Because the temperature is constant, the sensitivities of the species +concentrations with respect to the Arrhenius parameters (:math:`A_i` and :math:`E_i`) +become linearly dependent. This high correlation means the effects of the +pre-exponential factor and the activation energy cannot be uniquely distinguished +from the measurements. Consequently, the Fisher Information Matrix (FIM) becomes +ill-conditioned, resulting in a near-zero determinant and a very large condition number. + +To break this correlation and make the parameters identifiable, introducing a time- +varying temperature profile (for example, a temperature step or a ramp) is required. +As shown in the heatmap, when the initial temperature :math:`T_0` differs from the +subsequent 300 K baseline, such a temperature change breaks the linear dependence, +yielding a well-conditioned FIM and identifiable parameters. + + + +Step 6: Performing an optimal experimental design +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Step 5, we defined the ``run_reactor_doe`` function. This function constructs +the DoE object and performs the exploratory sensitivity analysis. The way the function +is defined, it also proceeds immediately to the optimal experimental design step +(applying ``run_doe`` on the ``DesignOfExperiments`` object). +We can initialize the model with the result we obtained from the exploratory +analysis (optimal point from the heatmaps) to help the optimal design step to speed +up convergence. However, implementation of this initialization is not shown here. + +After applying ``run_doe`` on the ``DesignOfExperiments`` object, +the optimal design is an initial concentration of 5.0 mol/L and +an initial temperature of 494 K with all other temperatures being 300 K. +The corresponding :math:`\log_{10}` determinant of the FIM is 19.32. diff --git a/doc/OnlineDocs/explanation/analysis/doe/multexp.rst b/doc/OnlineDocs/explanation/analysis/doe/multexp.rst new file mode 100644 index 00000000000..0c943b7e010 --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/doe/multexp.rst @@ -0,0 +1,8 @@ +.. _multexperiments: + +Multiple Experiments +==================== + +.. note:: + + Detailed descriptions and example code for simultaneous design of multiple experiments will be added in a future update. \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/doe/objective.rst b/doc/OnlineDocs/explanation/analysis/doe/objective.rst new file mode 100644 index 00000000000..28bd39972e9 --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/doe/objective.rst @@ -0,0 +1,8 @@ +.. _objectives: + +Objective Options +================= + +.. note:: + + Detailed descriptions and example code for the objective options in Pyomo.DoE will be added in a future update. \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/doe/overview.rst b/doc/OnlineDocs/explanation/analysis/doe/overview.rst new file mode 100644 index 00000000000..2c25f5d02fc --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/doe/overview.rst @@ -0,0 +1,131 @@ +Overview +======== + +Model-based Design of Experiments (MBDoE) is a technique to maximize the information +gain from experiments by directly using science-based models with physically meaningful +parameters. It is one key component in the model calibration and uncertainty +quantification workflow shown below: + +.. figure:: pyomo_workflow_new.png + :align: center + :scale: 90 % + +The parameter estimation, uncertainty analysis, and MBDoE are +combined into an iterative framework to select, refine, and calibrate science-based +mathematical models with quantified uncertainty. Currently, Pyomo.DoE focuses on +increasing parameter precision. + +Pyomo.DoE provides the exploratory analysis and MBDoE capabilities to the +Pyomo ecosystem. The user provides one Pyomo model, a set of parameter nominal values, +the allowable design spaces for design variables, and the assumed observation error model. +During exploratory analysis, Pyomo.DoE checks if the model parameters can be +inferred from the postulated measurements or preliminary data. +MBDoE then recommends optimized experimental conditions for collecting more data. +Parameter estimation packages such as :ref:`Parmest ` can perform +parameter estimation using the available data to infer values for parameters, +and facilitate an uncertainty analysis to approximate the parameter covariance matrix. +If the parameter uncertainties are sufficiently small, the workflow terminates +and returns the final model with quantified parametric uncertainty. +If not, MBDoE recommends optimized experimental conditions to generate new data +that will maximize information gain and eventually reduce parameter uncertainty. + +Below is an overview of the type of optimization models Pyomo.DoE can accommodate: + +* Pyomo.DoE is suitable for optimization models of **continuous** variables +* Pyomo.DoE can handle **equality constraints** defining state variables +* Pyomo.DoE supports (Partial) Differential-Algebraic Equations (PDAE) models via :ref:`Pyomo.DAE ` +* Pyomo.DoE also supports models with only algebraic equations + +The general form of a DAE problem that can be passed into Pyomo.DoE is shown below: + +.. math:: + :nowrap: + + \[\begin{array}{l} + \dot{\mathbf{x}}(t) = \mathbf{f}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}}, \boldsymbol{\theta}) \\ + \mathbf{g}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\theta})=\mathbf{0} \\ + \mathbf{y} =\mathbf{h}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\theta}) \\ + \mathbf{f}^{\mathbf{0}}\left(\dot{\mathbf{x}}\left(t_{0}\right), \mathbf{x}\left(t_{0}\right), \mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\theta}\right)=\mathbf{0} \\ + \mathbf{g}^{\mathbf{0}}\left( \mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\theta}\right)=\mathbf{0}\\ + \mathbf{y}^{\mathbf{0}}\left(t_{0}\right)=\mathbf{h}\left(\mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\theta}\right) + \end{array}\] + +where: + +* :math:`\boldsymbol{\theta} \in \mathbb{R}^{N_p}` are unknown model parameters. +* :math:`\mathbf{x} \subseteq \mathcal{X}` are dynamic state variables which characterize trajectory of the system, :math:`\mathcal{X} \in \mathbb{R}^{N_x \times N_t}`. +* :math:`\mathbf{z} \subseteq \mathcal{Z}` are algebraic state variables, :math:`\mathcal{Z} \in \mathbb{R}^{N_z \times N_t}`. +* :math:`\mathbf{u} \subseteq \mathcal{U}` are time-varying decision variables, :math:`\mathcal{U} \in \mathbb{R}^{N_u \times N_t}`. +* :math:`\overline{\mathbf{w}} \in \mathbb{R}^{N_w}` are time-invariant decision variables. +* :math:`\mathbf{y} \subseteq \mathcal{Y}` are measurement response variables, :math:`\mathcal{Y} \in \mathbb{R}^{N_r \times N_t}`. +* :math:`\mathbf{f}(\cdot)` are differential equations. +* :math:`\mathbf{g}(\cdot)` are algebraic equations. +* :math:`\mathbf{h}(\cdot)` are measurement functions. +* :math:`\mathbf{t} \in \mathbb{R}^{N_t \times 1}` is a union of all time sets. + +.. note:: + + Parameters and design variables should be defined as Pyomo ``Var`` components + when building the model using the ``Experiment`` class so that users can use both + ``Parmest`` and ``Pyomo.DoE`` seamlessly. + +Based on the above notation, the form of the MBDoE problem addressed in Pyomo.DoE is shown below: + +.. math:: + :nowrap: + + \begin{equation} + \begin{aligned} + \underset{\boldsymbol{\varphi}}{\max} \quad & \Psi (\mathbf{M}(\boldsymbol{\hat{\theta}}, \boldsymbol{\varphi})) \\ + \text{s.t.} \quad & \mathbf{M}(\boldsymbol{\hat{\theta}}, \boldsymbol{\varphi}) = \sum_r^{N_r} \sum_{r'}^{N_r} \tilde{\sigma}_{(r,r')}\mathbf{Q}_r^\mathbf{T} \mathbf{Q}_{r'} + \mathbf{M}_0 \\ + & \dot{\mathbf{x}}(t) = \mathbf{f}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}}) \\ + & \mathbf{g}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{y}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\hat{\theta}})=\mathbf{0} \\ + & \mathbf{y} =\mathbf{h}(\mathbf{x}(t), \mathbf{z}(t), \mathbf{u}(t), \overline{\mathbf{w}},\boldsymbol{\hat{\theta}}) \\ + & \mathbf{f}^{\mathbf{0}}\left(\dot{\mathbf{x}}\left(t_{0}\right), \mathbf{x}\left(t_{0}\right), \mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}})\right)=\mathbf{0} \\ + & \mathbf{g}^{\mathbf{0}}\left( \mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{y}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}}\right)=\mathbf{0}\\ + &\mathbf{y}^{\mathbf{0}}\left(t_{0}\right)=\mathbf{h}\left(\mathbf{x}\left(t_{0}\right),\mathbf{z}(t_0), \mathbf{u}\left(t_{0}\right), \overline{\mathbf{w}}, \boldsymbol{\hat{\theta}}\right) + \end{aligned} + \end{equation} + +where: + +* :math:`\boldsymbol{\varphi}` are design variables, which are manipulated to maximize the information content of experiments. It should consist of one or more of :math:`\mathbf{u}(t), \mathbf{y}^{\mathbf{0}}({t_0}),\overline{\mathbf{w}}`. With a proper model formulation, the timepoints for control or measurements :math:`\mathbf{t}` can also be degrees of freedom. +* :math:`\mathbf{M}` is the Fisher information matrix (FIM), approximated as the inverse of the covariance matrix of parameter estimates :math:`\boldsymbol{\hat{\theta}}`. A large FIM indicates more information contained in the experiment for parameter estimation. +* :math:`\mathbf{Q}` is the dynamic sensitivity matrix, containing the partial derivatives of :math:`\mathbf{y}` with respect to :math:`\boldsymbol{\theta}`. +* :math:`\Psi` is the scalar design criteria to measure the information content in the FIM. +* :math:`\mathbf{M}_0` is the sum of all the FIMs from previous experiments. + +Pyomo.DoE provides five design criteria :math:`\Psi` to measure the information in the FIM. +The covariance matrix of parameter estimates is approximated as the inverse of the FIM, +i.e., :math:`\mathbf{V} \approx \mathbf{M}^{-1}`. +We can use the FIM or the covariance matrix to define the design criteria. + +.. list-table:: Pyomo.DoE design criteria + :header-rows: 1 + :class: tight-table + + * - Design criterion + - Computation + - Geometrical meaning + * - A-optimality + - :math:`\text{trace}(\mathbf{V}) = \text{trace}(\mathbf{M}^{-1})` + - Minimizing this is equivalent to minimizing the enclosing box of the confidence ellipse + * - Pseudo A-optimality + - :math:`\text{trace}(\mathbf{M})` + - Maximizing this is equivalent to maximizing the dimensions of the enclosing box of the Fisher Information Matrix + * - D-optimality + - :math:`\det(\mathbf{M}) = 1/\det(\mathbf{V})` + - Maximizing this is equivalent to minimizing confidence-ellipsoid volume + * - E-optimality + - :math:`\lambda_{\min}(\mathbf{M}) = 1/\lambda_{\max}(\mathbf{V})` + - Maximizing this is equivalent to minimizing the longest axis of the confidence ellipse + * - Modified E-optimality + - :math:`\text{cond}(\mathbf{M}) = \text{cond}(\mathbf{V})` + - Minimizing this is equivalent to minimizing the ratio of the longest axis to the shortest axis of the confidence ellipse + +.. note:: + + A confidence ellipse is a geometric representation of the uncertainty in parameter + estimates. It is derived from the covariance matrix :math:`\mathbf{V}`. + +In order to solve problems of the above, Pyomo.DoE implements the 2-stage stochastic program. Please see Wang and Dowling (2022) for details. \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/doe/uncertainty.rst b/doc/OnlineDocs/explanation/analysis/doe/uncertainty.rst new file mode 100644 index 00000000000..50b31dbe040 --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/doe/uncertainty.rst @@ -0,0 +1,10 @@ +.. _paramuncertainty: + +Parameter Uncertainty +===================== + +.. note:: + + Detailed descriptions and example code for experiment design under parameter uncertainty will be added in a + future update. + diff --git a/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst b/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst index b7142e0e103..971c411c358 100644 --- a/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst +++ b/doc/OnlineDocs/explanation/analysis/parmest/covariance.rst @@ -1,14 +1,20 @@ .. _covariancesection: +Uncertainty Quantification +========================== +The goal of parameter estimation (see :ref:`driversection` Section) is to estimate unknown model parameters +from experimental data. Uncertainty quantification is required to ensure that the estimates of the parameters are +close to their true values. This parameter uncertainty can be computed using four methods in parmest: +covariance matrix, likelihood ratio test, bootstrapping, and leave-N-out. + Covariance Matrix Estimation -============================ +---------------------------- -The goal of parameter estimation (see :ref:`driversection` Section) is to estimate unknown model parameters -from experimental data. When the model parameters are estimated from the data, their accuracy is measured by -computing the covariance matrix. The diagonal of this covariance matrix contains the variance of the -estimated parameters which is used to calculate their uncertainty. Assuming Gaussian independent and identically -distributed measurement errors, the covariance matrix of the estimated parameters can be computed using the -following methods which have been implemented in parmest. +The uncertainty in estimated model parameters can be quantified by computing the covariance matrix. +The diagonal of this covariance matrix contains the variance of the estimated parameters which is used to +calculate their uncertainty. Assuming Gaussian independent and identically distributed measurement errors, +the covariance matrix of the estimated parameters can be computed using the following methods which have been +implemented in parmest. 1. Reduced Hessian Method @@ -138,4 +144,90 @@ e.g., >>> pest = parmest.Estimator(exp_list, obj_function="SSE") >>> obj_val, theta_val = pest.theta_est() >>> cov_method = "reduced_hessian" - >>> cov = pest.cov_est(method=cov_method) \ No newline at end of file + >>> cov = pest.cov_est(method=cov_method) + +Bootstrapping +------------- + +Bootstrapping is a non-intrusive method that uses resampling to approximate the variance of the parameter estimates. +By repeatedly fitting the model to resampled datasets, the parameter uncertainty (e.g., variance or covariance) +can be computed without relying on strong distributional assumptions. This method is summarized as follows: + +1. Step 0: Define the input data + + Given input data: :math:`[y_1, \dots, y_n]` + +2. Step 1: Generate :math:`B` artificial datasets through sampling + + Sample with replacement from the original data to create :math:`B` bootstrap datasets: + + .. math:: + \left\{y_1^{(1)}, \dots, y_n^{(1)} \right\}, \dots, \left\{y_1^{(B)}, \dots, y_n^{(B)} \right\} + +3. Step 2: Compute the estimator of the parameters + + Fit the model to each bootstrap dataset to obtain parameter estimates: + + .. math:: + \left\{ \hat{\theta}^{(1)}, \dots, \hat{\theta}^{(B)} \right\} + +4. Step 3: Compute the approximate variance of the parameter estimates + + The variability across bootstrap estimates approximates the estimator variance: + + .. math:: + \text{Var}(\hat{\theta}) = + \frac{1}{B} \sum_{j=1}^{B} \left(\hat{\theta}^{(j)}\right)^2 + - + \left( + \frac{1}{B} \sum_{j=1}^{B} \hat{\theta}^{(j)} + \right)^2 + +.. note:: + + The example code for this method will soon be provided. + +Likelihood Ratio Test +--------------------- + +The likelihood ratio test is a non-intrusive method that compares how well two parameter sets explain the +observed data: an unconstrained set and a constrained set defined by the null hypothesis. It is commonly used +to assess whether restricting parameters significantly degrades model fit. This method is summarized as follows: + +1. Step 1: State the hypothesis + + Null hypothesis: :math:`\theta \in \Theta_0` vs. Alternative hypothesis: :math:`\theta \notin \Theta_0` + +2. Step 2: Define the test statistic + + The test statistic is the ratio of the maximum likelihood under the full parameter space to that under the + constrained space: + + .. math:: + \lambda_n = + \frac{\sup_{\theta \in \Theta} L(\theta; y_1, \dots, y_n)}{ + \sup_{\theta \in \Theta_0} L(\theta; y_1, \dots, y_n)} + +3. Step 3: Define the decision rule + + Reject the null hypothesis if: + + .. math:: + \lambda_n > c_{\alpha} + + Equivalently, using maximum likelihood estimates: + + .. math:: + \lambda_n = + \frac{L(\hat{\theta}; y_1, \dots, y_n)}{L(\hat{\theta}_0; y_1, \dots, y_n)} > c_{\alpha} + +.. note:: + + The example code for this method will soon be provided. + +Leave-N-Out +----------- + +.. note:: + + Detailed descriptions and example code for this method will be added in a future update. \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/parmest/estimability.rst b/doc/OnlineDocs/explanation/analysis/parmest/estimability.rst new file mode 100644 index 00000000000..4cf40728689 --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/parmest/estimability.rst @@ -0,0 +1,55 @@ +.. _estimabilitysection: + +Estimability Analysis +===================== + +After estimating the model parameters with their associated uncertainty, as demonstrated in the +:ref:`driversection` and :ref:`covariancesection` Section, estimability analysis is required to identify +parameters that cannot be reliably estimated from the available data due to limitations in the mathematical +model structure. If such parameters are identified, the model may need to be reformulated, replaced with an +alternative structure, or augmented with additional prior information. In parmest, estimability analysis can +be performed using eigen-decomposition of the parameter covariance matrix, profile likelihood methods, or +multi-start initialization routines. + +Eigen-decomposition +------------------- + +The estimability of model parameters can be analyzed through eigen-decomposition of the covariance matrix +obtained from parameter estimation. This covariance matrix quantifies parameter uncertainty and captures both +parameter variances and correlations. Eigen-decomposition of this matrix identifies principal directions +in parameter space along which uncertainty is largest or smallest. These directions provide insight into +parameter identifiability and reveal combinations of parameters that are either structurally identifiable or +non-identifiable based on the underlying model formulation. + +.. note:: + + Detailed descriptions and example code for this method will be added in a future update. + +Profile Likelihood +------------------ + +Profile likelihood analysis evaluates parameter estimability by systematically varying one parameter while +re-optimizing the remaining parameters to maintain consistency with the model and observed data. This approach +is closely related to likelihood ratio–based uncertainty quantification and provides a robust characterization +of practical identifiability through the shape of the likelihood surface. In addition, it can reveal structural +non-identifiability when flat or unbounded profiles indicate parameter combinations that are not uniquely +determined by the model formulation, particularly in nonlinear systems. + +.. note:: + + Detailed descriptions and example code for this method will be added in a future update. + +Multi-start Initialization +-------------------------- + +Multi-start initialization assesses parameter estimability by exploring a range of initial guesses. Because +parameter estimation problems are often nonlinear and may exhibit multiple local minima, different initializations +can lead to different parameter estimates. By solving the estimation problem from multiple starting points, one can +evaluate the robustness of the solution and identify potential issues related to non-convexity or non-identifiability. +Consistent convergence to a unique solution across initializations suggests that the parameters are structurally +identifiable within the model formulation, whereas sensitivity to initialization indicates potential estimability +issues. + +.. note:: + + Detailed descriptions and example code for this method will be added in a future update. \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/parmest/index.rst b/doc/OnlineDocs/explanation/analysis/parmest/index.rst index 4616a2134d4..54e85a67071 100644 --- a/doc/OnlineDocs/explanation/analysis/parmest/index.rst +++ b/doc/OnlineDocs/explanation/analysis/parmest/index.rst @@ -27,4 +27,6 @@ Index of parmest documentation graphics.rst examples.rst parallel.rst + objective.rst + estimability.rst api.rst diff --git a/doc/OnlineDocs/explanation/analysis/parmest/objective.rst b/doc/OnlineDocs/explanation/analysis/parmest/objective.rst new file mode 100644 index 00000000000..ccce0eb8ac2 --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/parmest/objective.rst @@ -0,0 +1,8 @@ +.. _objectivesection: + +Objective Options +================= + +.. note:: + + Detailed descriptions and example code for the objective options in parmest will be added in a future update. \ No newline at end of file diff --git a/doc/OnlineDocs/explanation/analysis/parmest/overview.rst b/doc/OnlineDocs/explanation/analysis/parmest/overview.rst index 7f6f23c9140..7a3f7538fde 100644 --- a/doc/OnlineDocs/explanation/analysis/parmest/overview.rst +++ b/doc/OnlineDocs/explanation/analysis/parmest/overview.rst @@ -12,10 +12,12 @@ Functionality in parmest includes: * Model-based parameter estimation using experimental data * Covariance matrix estimation -* Bootstrap resampling for parameter estimation +* Bootstrap resampling for uncertainty quantification * Confidence regions based on single or multi-variate distributions -* Likelihood ratio +* Likelihood ratio test * Leave-N-out cross validation +* Regularization for objective function improvement +* Multi-start initialization optimization * Parallel processing Background diff --git a/pyomo/contrib/doe/__init__.py b/pyomo/contrib/doe/__init__.py index 53b532506a2..3baf8a4c62d 100644 --- a/pyomo/contrib/doe/__init__.py +++ b/pyomo/contrib/doe/__init__.py @@ -6,8 +6,8 @@ # Solutions of Sandia, LLC, the U.S. Government retains certain rights in this # software. This software is distributed under the 3-clause BSD License. # ____________________________________________________________________________________ -from .doe import DesignOfExperiments, ObjectiveLib, FiniteDifferenceStep -from .utils import rescale_FIM +from .doe import DesignOfExperiments, ObjectiveLib, FiniteDifferenceStep, GradientMethod +from .utils import rescale_FIM, ExperimentGradients from .grey_box_utilities import FIMExternalGreyBox # Deprecation errors for old Pyomo.DoE interface classes and structures diff --git a/pyomo/contrib/doe/documentation.md b/pyomo/contrib/doe/documentation.md new file mode 100644 index 00000000000..e92d4d91a9d --- /dev/null +++ b/pyomo/contrib/doe/documentation.md @@ -0,0 +1,187 @@ +# DesignOfExperiments `optimize_experiments()` Proposed Interface + +This is a brief documentation explaining the changes. This document will not get merged. + +## Why This Change + +The `optimize_experiments()` path has grown from a single-experiment workflow into a +multi-experiment optimization interface with optional initialization strategies and +richer result payloads. This document captures the interface contract, operating modes, +and optimization model at a high level. + +## API Summary + +### Constructor + +```python +DesignOfExperiments( + experiment=..., # single experiment object OR list of experiment objects + ... +) +``` + +- `experiment` now accepts either: + - one experiment object (template mode input), or + - a list of experiment objects (user-initialized mode input). +- Internally, the implementation normalizes this to `self.experiment_list`. + +### Multi-Experiment Solve Entry Point + +```python +optimize_experiments( + results_file=None, + n_exp: int = None, + init_method: InitializationMethod = None, + init_n_samples: int = 5, + init_seed: int = None, + init_parallel: bool = False, + init_combo_parallel: bool = False, + init_n_workers: int = None, + init_combo_chunk_size: int = 5000, + init_combo_parallel_threshold: int = 20000, + init_max_wall_clock_time: float = None, + init_solver=None, +) +``` + +### Short description of arguments + +- `results_file`: Optional path for writing JSON results. +- `n_exp`: Number of experiments to optimize in template mode. +- `init_method`: Initialization strategy (`None` or `"lhs"`). +- `init_n_samples`: Number of LHS samples per experiment-input dimension. +- `init_seed`: Random seed used by LHS initialization. +- `init_parallel`: Enables parallel candidate-point FIM evaluation. +- `init_combo_parallel`: Enables parallel candidate-combination scoring. +- `init_n_workers`: Worker count for LHS parallel evaluation/scoring paths. +- `init_combo_chunk_size`: Number of combinations handled per worker task. +- `init_combo_parallel_threshold`: Minimum combinations required before using combo parallelism. +- `init_max_wall_clock_time`: Optional LHS time budget (seconds); returns best-so-far if exceeded. +- `init_solver`: Optional solver used only during initialization phases (LHS and square init solve). + +## Operating Modes + +### 1) Template Mode + +- Condition: `len(experiment) == 1` +- `n_exp` may be provided to choose how many experiments to optimize simultaneously. +- If `n_exp` is omitted, default is `1`. + +### 2) User-Initialized Mode + +- Condition: `len(experiment) > 1` +- Number of experiments is fixed by the list length. +- `n_exp` must not be provided. + +## Initialization Behavior + +### No Special Initialization (`init_method=None`) + +- Uses current experiment design values from the model labels directly. + +### LHS Initialization (`init_method="lhs"`) + +- Currently supported in template mode. +- Requires explicit lower and upper bounds for all experiment inputs. +- Generates candidate points using per-dimension 1-D LHS, then Cartesian product. +- Scores combinations of candidate points using objective-specific FIM metrics. +- Selects best-scoring set of initial points for the nonlinear solve. + +### `init_solver` (new) + +- If provided, `init_solver` is used for: + - initialization-phase solves (including multi-experiment block-construction + solves and the LHS candidate FIM evaluation path), + - the square initialization solve before final optimization. +- Final optimization solve still uses the main DoE solver (`self.solver`). +- If `init_solver` is `None`, initialization also uses `self.solver`. + +## Optimization Formulation + +The proposed multi-experiment interface follows a simultaneous design formulation. + +### General Form + +Let: + +- $E = \{1, 2, ..., N_{exp}\}$ be the experiment index set, +- $\phi_k$ be the design variables for experiment `k`, +- $\theta$ be the model parameters +- $M_0$ be the prior Fisher Information Matrix (FIM), +- $M_k(\hat{\theta}, \phi_k)$ be the FIM contribution from experiment `k`, +- $\Psi(M)$ be the chosen FIM metric (D-, A-, pseudo-A-, etc.). + +Then: + +```math +\max_{\phi_1,\ldots,\phi_{N_{exp}}} \Psi(\mathbf{M}) +``` + +subject to: + +```math +\mathbf{M} = \sum_{k=1}^{N_{exp}} \mathbf{M}_k(\hat{\theta}, \phi_k) + \mathbf{M}_0 +``` + +```math +\mathbf{M}_k = \mathbf{Q}_k^\top \Sigma_{\bar{y},k}^{-1} \mathbf{Q}_k, \quad \forall k \in E +``` + +```math +\mathbf{m}(\bar{x}_k, \hat{\bar{y}}_k, \phi_k, \hat{\theta}, t) = 0, \quad \forall k \in E +``` + +```math +\mathbf{g}(\bar{x}_k, \hat{\bar{y}}_k, \phi_k, \hat{\theta}, t) \le 0, \quad \forall k \in E +``` + +where $\mathbf{Q}_k$ is the sensitivity matrix for experiment `k`. + +### Symmetry-Breaking Constraint + +To avoid permutation-equivalent solutions in simultaneous design: + +```math +\varphi_{\text{primary},1} \le \varphi_{\text{primary},2} \le \cdots \le \varphi_{\text{primary},N_{exp}} +``` +Here, $\varphi_{\text{primary},\cdot}$ is the primary variable which is considered for symmetry breaking. + +This is implemented in `optimize_experiments()` by using a user-marked primary design +variable passed in a `Pyomo.Suffix` (or a default selection (first variable from +experiment_inputs Suffix) with warning if not marked). + +### Current Implementation Specialization + +The current implementation corresponds to the single-scenario case (`N_s = 1`) with: + +```math +\mathbf{M}_{\text{total}} = \mathbf{M}_0 + \sum_{k=1}^{N_{exp}} \mathbf{M}_k +``` +Future implementation will handle parametric uncertainty with $N_s >1$ + +Objective handling in code: + +- Determinant and pseudo-trace: solved in maximization form (with monotonic transforms used in the NLP objective expressions). +- Trace: solved in minimization form via covariance/FIM-inverse representation. +- Zero objective: feasibility/debug mode. + +Cholesky-based constraints and variables are used in supported objective paths to stabilize determinant/trace formulations. + +We are also implementing `GreyBox-based` A, D, E, and ME-optimality objectives. + +## Result Highlights + +The solver output includes both legacy fields and structured fields: + +- `run_info` (API name, solver status/termination info) +- `settings` (objective, finite-difference, initialization config, modeling mode) +- `timing` (build/initialization/solve/total timing) +- `names` (design/output/parameter/error labels) +- `diagnostics` (symmetry and LHS diagnostics) +- `param_scenarios` (scenario-level objective metrics, total FIM, per-experiment details) + +Notably, initialization settings now include: + +- `settings["initialization"]["solver_name"]` + +to make it explicit which solver was used during initialization. diff --git a/pyomo/contrib/doe/doe.py b/pyomo/contrib/doe/doe.py index a13e6785bf4..2a967d6e0ec 100644 --- a/pyomo/contrib/doe/doe.py +++ b/pyomo/contrib/doe/doe.py @@ -25,11 +25,20 @@ # ____________________________________________________________________________________ from enum import Enum -from itertools import permutations, product - +from itertools import ( + permutations, + product, + combinations as _combinations, + islice as _islice, +) +import concurrent.futures as _cf import json import logging import math +import os +import threading +import time +import warnings from pyomo.common.dependencies import ( numpy as np, @@ -49,13 +58,16 @@ from pyomo.contrib.doe.grey_box_utilities import FIMExternalGreyBox from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxBlock + from scipy.stats.qmc import LatinHypercube import pyomo.environ as pyo from pyomo.contrib.doe.utils import ( check_FIM, compute_FIM_metrics, _SMALL_TOLERANCE_DEFINITENESS, + ExperimentGradients, ) +from pyomo.contrib.parmest.utils.model_utils import update_model_from_suffix from pyomo.opt import SolverStatus @@ -75,10 +87,48 @@ class FiniteDifferenceStep(Enum): backward = "backward" +class GradientMethod(Enum): + """Available sensitivity-calculation backends for DoE workflows.""" + + forward = "forward" + central = "central" + backward = "backward" + pynumero = "pynumero" + kaug = "kaug" + + +class InitializationMethod(Enum): + lhs = "lhs" + + +class _DoEResultsJSONEncoder(json.JSONEncoder): + """JSON encoder for DoE result payloads with numpy/Pyomo objects.""" + + def default(self, obj): + if isinstance(obj, np.generic): + return obj.item() + if isinstance(obj, np.ndarray): + return obj.tolist() + if isinstance(obj, Enum): + return str(obj) + return super().default(obj) + + class DesignOfExperiments: + # Objective options whose scalar score is compared with "larger is better" + # in initialization and diagnostics paths. + _MAXIMIZE_OBJECTIVES = frozenset( + { + ObjectiveLib.determinant, + ObjectiveLib.pseudo_trace, + ObjectiveLib.minimum_eigenvalue, + } + ) + def __init__( self, - experiment=None, + experiment: list = None, + gradient_method=None, fd_formula="central", step=1e-3, objective_option="determinant", @@ -109,14 +159,24 @@ def __init__( Parameters ---------- experiment: - Experiment object that holds the model and labels all the components. The - object should have a ``get_labeled_model`` where a model is returned with - the following labeled sets: + Experiment object(s) that hold the model and labels all the components. + Can be a single Experiment object or a list of Experiment objects. + For single experiments, you can pass the object directly: + ``experiment=exp`` or as a list: ``experiment=[exp]``. + Each object should have a ``get_labeled_model`` method that returns a model with + the following labeled Pyomo Suffixes: - ``unknown_parameters``, - ``experimental_inputs``, - ``experimental_outputs`` - + - ``measurement_error``. + + gradient_method: + Sensitivity-calculation backend used by the DoE workflow. Options + are ``forward``, ``central``, ``backward``, ``pynumero``, and + ``kaug``. If omitted, the value from ``fd_formula`` is used for + backward compatibility with the existing finite-difference + interface. fd_formula: Finite difference formula for computing the sensitivity matrix. Must be one of [``central``, ``forward``, ``backward``], default: ``central`` @@ -129,7 +189,8 @@ def __init__( - ``determinant`` (for determinant, or D-optimality), - ``trace`` (for trace of covariance matrix, or A-optimality), - ``pseudo_trace`` (for trace of Fisher Information Matrix(FIM), or pseudo A-optimality), - - ``minimum_eigenvalue``, (for E-optimality), or ``condition_number`` (for ME-optimality) + - ``minimum_eigenvalue``, (for E-optimality), or + - ``condition_number`` (for ME-optimality) Note: E-optimality and ME-optimality are only supported when using the grey box objective (i.e., ``grey_box_solver`` is True) default: ``determinant`` @@ -186,20 +247,48 @@ def __init__( Specify the level of the logger. Change to logging.DEBUG for all messages. """ + # Validate experiment(s) are provided if experiment is None: - raise ValueError("Experiment object must be provided to perform DoE.") - - # Check if the Experiment object has callable ``get_labeled_model`` function - if not hasattr(experiment, "get_labeled_model"): raise ValueError( - "The experiment object must have a ``get_labeled_model`` function" + "The 'experiment' parameter is required. " + "Pass a single Experiment object or a list of Experiment objects." ) - # Set the experiment object from the user - self.experiment = experiment + # Auto-convert single experiment to list + if not isinstance(experiment, list): + experiment_list = [experiment] + else: + experiment_list = experiment + + # Validate list is not empty + if len(experiment_list) == 0: + raise ValueError("The 'experiment' list cannot be empty.") - # Set the finite difference and subsequent step size - self.fd_formula = FiniteDifferenceStep(fd_formula) + # Check each experiment has get_labeled_model method + for idx, exp in enumerate(experiment_list): + if not hasattr(exp, "get_labeled_model"): + raise ValueError( + f"Experiment at index {idx} in 'experiment' must have a " + f"'get_labeled_model' method" + ) + + # Store experiment_list + self.experiment_list = experiment_list + + # Set the gradient method and finite difference formula + if gradient_method is None: + self._gradient_method = GradientMethod(fd_formula) + else: + self._gradient_method = GradientMethod(gradient_method) + + if self._gradient_method in ( + GradientMethod.forward, + GradientMethod.central, + GradientMethod.backward, + ): + self.fd_formula = FiniteDifferenceStep(self._gradient_method.value) + else: + self.fd_formula = None self.step = step # Set the objective type and scaling options: @@ -269,6 +358,63 @@ def __init__( # (i.e., no model rebuilding for large models with sequential) self._built_scenarios = False + @staticmethod + def _enum_label(value): + """Return a stable short label for enum-like values.""" + return str(value).split(".")[-1] + + def _grey_box_output_name(self): + """Return the output label exposed by the FIM grey box model.""" + if self.objective_option == ObjectiveLib.trace: + return "A-opt" + if self.objective_option == ObjectiveLib.determinant: + return "log-D-opt" + if self.objective_option == ObjectiveLib.minimum_eigenvalue: + return "E-opt" + if self.objective_option == ObjectiveLib.condition_number: + return "ME-opt" + raise ValueError( + "Grey-box objective support is only available for " + "objective_option in ['determinant', 'trace', " + "'minimum_eigenvalue', 'condition_number']." + ) + + def _initialize_grey_box_block(self, egb_block, fim_np, parameter_names): + """ + Seed a grey box block from the FIM computed by the square solve. + + The square solve initializes the finite-difference scenarios and the + aggregated FIM, but the external grey box block stores that FIM through + its own input/output variables. We therefore push the current FIM values + into the grey box explicitly before the final NLP solve so the external + model starts from a consistent state. + """ + param_list = list(parameter_names) + + # The external model only takes the symmetric triangular FIM entries as + # inputs. That keeps the grey box interface compact while still + # reconstructing the full symmetric FIM internally. + for i, p1 in enumerate(param_list): + for j, p2 in enumerate(param_list): + if i >= j: + egb_block.inputs[(p2, p1)].set_value(float(fim_np[i, j])) + + # Initialize the single grey box output so the final solve begins with + # an objective value consistent with the current square-solve FIM. + if self.objective_option == ObjectiveLib.trace: + output_value = np.trace(np.linalg.pinv(fim_np)) + elif self.objective_option == ObjectiveLib.determinant: + output_value = np.log(np.linalg.det(fim_np)) + elif self.objective_option == ObjectiveLib.minimum_eigenvalue: + output_value = np.min(np.linalg.eigvalsh(fim_np)) + elif self.objective_option == ObjectiveLib.condition_number: + eig = np.linalg.eigvalsh(fim_np) + output_value = np.log(np.abs(np.max(eig) / np.min(eig))) + else: + output_value = 0.0 + + egb_block.outputs[self._grey_box_output_name()].set_value(float(output_value)) + # Perform doe def run_doe(self, model=None, results_file=None): """ @@ -283,13 +429,18 @@ def run_doe(self, model=None, results_file=None): default: None --> don't save """ + if self._gradient_method == GradientMethod.kaug: + raise ValueError( + "Cannot use GradientMethod.kaug for DoE optimization. " + "Use a finite-difference method or GradientMethod.pynumero." + ) + # Check results file name if results_file is not None: - if type(results_file) not in [pathlib.Path, str]: + if not isinstance(results_file, (pathlib.Path, str)): raise ValueError( "``results_file`` must be either a Path object or a string." ) - # Start timer sp_timer = TicTocTimer() sp_timer.tic(msg=None) @@ -331,7 +482,7 @@ def run_doe(self, model=None, results_file=None): # and fix the design variables. model.objective.deactivate() model.obj_cons.deactivate() - for comp in model.scenario_blocks[0].experiment_inputs: + for comp in model.fd_scenario_blocks[0].experiment_inputs: comp.fix() # TODO: safeguard solver call to see if solver terminated successfully @@ -357,37 +508,17 @@ def run_doe(self, model=None, results_file=None): model.dummy_obj.deactivate() # Reactivate objective and unfix experimental design decisions - for comp in model.scenario_blocks[0].experiment_inputs: + for comp in model.fd_scenario_blocks[0].experiment_inputs: comp.unfix() model.objective.activate() model.obj_cons.activate() if self.use_grey_box: - # Initialize grey box inputs to be fim values currently - for i in model.parameter_names: - for j in model.parameter_names: - if list(model.parameter_names).index(i) >= list( - model.parameter_names - ).index(j): - model.obj_cons.egb_fim_block.inputs[(j, i)].set_value( - pyo.value(model.fim[(i, j)]) - ) - # Set objective value - if self.objective_option == ObjectiveLib.trace: - trace_val = np.trace(np.linalg.pinv(self.get_FIM())) - model.obj_cons.egb_fim_block.outputs["A-opt"].set_value(trace_val) - elif self.objective_option == ObjectiveLib.determinant: - det_val = np.linalg.det(np.array(self.get_FIM())) - model.obj_cons.egb_fim_block.outputs["log-D-opt"].set_value( - np.log(det_val) - ) - elif self.objective_option == ObjectiveLib.minimum_eigenvalue: - eig, _ = np.linalg.eig(np.array(self.get_FIM())) - model.obj_cons.egb_fim_block.outputs["E-opt"].set_value(np.min(eig)) - elif self.objective_option == ObjectiveLib.condition_number: - eig, _ = np.linalg.eig(np.array(self.get_FIM())) - cond_number = np.log(np.abs(np.max(eig) / np.min(eig))) - model.obj_cons.egb_fim_block.outputs["ME-opt"].set_value(cond_number) + self._initialize_grey_box_block( + model.obj_cons.egb_fim_block, + np.asarray(self.get_FIM(model=model), dtype=np.float64), + model.parameter_names, + ) # If the model has L, initialize it with the solved FIM if hasattr(model, "L"): @@ -409,7 +540,7 @@ def run_doe(self, model=None, results_file=None): # Check if the FIM is positive definite # If not, add jitter to the diagonal # to ensure positive definiteness - min_eig = np.min(np.linalg.eigvals(fim_np)) + min_eig = np.min(np.real(np.linalg.eigvals(fim_np))) if min_eig < _SMALL_TOLERANCE_DEFINITENESS: # Raise the minimum eigenvalue to at @@ -478,10 +609,14 @@ def run_doe(self, model=None, results_file=None): self.results["Solver Status"] = res.solver.status self.results["Termination Condition"] = res.solver.termination_condition - if type(res.solver.message) is str: + if isinstance(res.solver.message, str): results_message = res.solver.message - elif type(res.solver.message) is bytes: + elif isinstance(res.solver.message, bytes): results_message = res.solver.message.decode("utf-8") + else: + results_message = ( + str(res.solver.message) if res.solver.message is not None else "" + ) self.results["Termination Message"] = results_message # Important quantities for optimal design @@ -489,29 +624,29 @@ def run_doe(self, model=None, results_file=None): self.results["Sensitivity Matrix"] = self.get_sensitivity_matrix() self.results["Experiment Design"] = self.get_experiment_input_values() self.results["Experiment Design Names"] = [ - str(pyo.ComponentUID(k, context=model.scenario_blocks[0])) - for k in model.scenario_blocks[0].experiment_inputs + str(pyo.ComponentUID(k, context=model.fd_scenario_blocks[0])) + for k in model.fd_scenario_blocks[0].experiment_inputs ] self.results["Experiment Outputs"] = self.get_experiment_output_values() self.results["Experiment Output Names"] = [ - str(pyo.ComponentUID(k, context=model.scenario_blocks[0])) - for k in model.scenario_blocks[0].experiment_outputs + str(pyo.ComponentUID(k, context=model.fd_scenario_blocks[0])) + for k in model.fd_scenario_blocks[0].experiment_outputs ] self.results["Unknown Parameters"] = self.get_unknown_parameter_values() self.results["Unknown Parameter Names"] = [ - str(pyo.ComponentUID(k, context=model.scenario_blocks[0])) - for k in model.scenario_blocks[0].unknown_parameters + str(pyo.ComponentUID(k, context=model.fd_scenario_blocks[0])) + for k in model.fd_scenario_blocks[0].unknown_parameters ] self.results["Measurement Error"] = self.get_measurement_error_values() self.results["Measurement Error Names"] = [ - str(pyo.ComponentUID(k, context=model.scenario_blocks[0])) - for k in model.scenario_blocks[0].measurement_error + str(pyo.ComponentUID(k, context=model.fd_scenario_blocks[0])) + for k in model.fd_scenario_blocks[0].measurement_error ] - self.results["Prior FIM"] = [list(row) for row in list(self.prior_FIM)] + self.results["Prior FIM"] = self.prior_FIM.tolist() # Saving some stats on the FIM for convenience - self.results["Objective expression"] = str(self.objective_option).split(".")[-1] + self.results["Objective expression"] = self._enum_label(self.objective_option) self.results["log10 A-opt"] = np.log10(np.trace(np.linalg.inv(fim_local))) self.results["log10 pseudo A-opt"] = np.log10(np.trace(fim_local)) self.results["log10 D-opt"] = np.log10(np.linalg.det(fim_local)) @@ -525,7 +660,8 @@ def run_doe(self, model=None, results_file=None): self.results["Wall-clock Time"] = build_time + initialization_time + solve_time # Settings used to generate the optimal DoE - self.results["Finite Difference Scheme"] = str(self.fd_formula).split(".")[-1] + self.results["Gradient Method"] = self._enum_label(self._gradient_method) + self.results["Finite Difference Scheme"] = self._enum_label(self.fd_formula) self.results["Finite Difference Step"] = self.step self.results["Nominal Parameter Scaling"] = self.scale_nominal_param_value @@ -537,6 +673,1618 @@ def run_doe(self, model=None, results_file=None): with open(results_file, "w") as file: json.dump(self.results, file) + def optimize_experiments( + self, + results_file=None, + n_exp: int = None, + init_method: InitializationMethod = None, + init_n_samples: int = 5, + init_seed: int = None, + init_parallel: bool = False, + init_combo_parallel: bool = False, + init_n_workers: int = None, + init_combo_chunk_size: int = 5000, + init_combo_parallel_threshold: int = 20000, + init_max_wall_clock_time: float = None, + init_solver=None, + ): + """ + Optimize single experiment or multiple experiments simultaneously for + Design of Experiments. + + The number of experiments is determined by the length of the + experiment list provided through the ``experiment`` argument. + + Parameters + ---------- + results_file: + string name of the file path to save the results to in the form + of a .json file + + init_method: + Method used to initialize the experiment design variables before + optimization. Options are: + + - ``None`` (default): No special initialization; use the initial + values from ``get_labeled_model()``. To provide a custom starting + point, initialize the ``Experiment`` objects with the desired + design values before passing them in ``experiment``. + - ``"lhs"`` (or ``InitializationMethod.lhs``): Use Latin Hypercube Sampling (LHS) to find a good + initial design. For each experiment-input dimension, ``init_n_samples`` + points are sampled independently using 1-D LHS, and their Cartesian + product forms the set of candidate experiment designs. The FIM is + evaluated at every candidate, and the combination of ``n_exp`` + candidates (without replacement) that best satisfies the chosen + objective is selected as the starting point for the optimization. + + init_n_samples: + Number of LHS samples per experiment-input dimension when + ``init_method="lhs"``. The total number of candidate + designs is ``init_n_samples ** n_exp_inputs``. A warning is issued + when this exceeds 10,000. Default: 5. + + init_seed: + Integer seed for the LHS random-number generator (for + reproducibility). Used only when ``init_method="lhs"``. + Default: ``None`` (non-deterministic). + init_parallel: + If True, evaluate candidate-point FIMs in parallel during LHS + initialization. Default: False. + init_combo_parallel: + If True, the scoring of Latin hypercube candidate combinations + (``C(n_candidates, n_exp)`` during ``init_method="lhs"``) + is split across a thread pool. Each worker computes the scalar + objective derived from the FIM for its chunk of combinations. The + flag has no effect unless ``init_method="lhs"`` and the + total number of combinations exceeds ``init_combo_parallel_threshold``. + Default: False. + init_n_workers: + Number of worker threads for combination FIM metric when + ``init_combo_parallel=True``. Default: ``None`` (auto-select). + init_combo_chunk_size: + Number of combinations scored per worker task. Default: 5000. + init_combo_parallel_threshold: + Parallel combo scoring is used only when number of combinations is + at least this value. Default: 20000. + init_max_wall_clock_time: + Optional time budget (seconds) for LHS initialization. If exceeded + during combination scoring, best-so-far is returned. + init_solver: + Optional solver object used only for initialization phases + (including multi-experiment block-construction solves, LHS + candidate-FIM evaluations, and the square initialization solve). + The final optimization solve always uses the primary DoE solver + (``self.solver``). If ``init_solver = None``, initialization also uses + ``self.solver``. + + Notes + ----- + Number of Experiments: + When ``len(experiment) == 1`` (template mode), pass ``n_exp`` + to specify how many experiments to optimize. When + ``len(experiment) > 1`` (user-initialized mode), the list + length determines the number of experiments and ``n_exp`` must + not be set. + + Symmetry Breaking (for multiple experiments): + To prevent equivalent permutations of identical experiments, you must + mark a "primary" design variable using a Pyomo Suffix in your experiment's + `label_experiment()` method: + + Example:: + + m.sym_break_cons = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.sym_break_cons[m.CA[0]] = None # Mark CA[0] as primary variable + + This will add constraints: exp[k-1].primary_var <= exp[k].primary_var + for k = 1, ..., n_exp-1, which breaks permutation symmetry and can + significantly reduce solve times. + + LHS Initialization (init_method="lhs"): + Each dimension of the experiment inputs is sampled independently + using a 1-D Latin Hypercube, giving ``init_n_samples`` evenly-spaced + stratified samples across the variable bounds. The joint candidate + set is the Cartesian product of these per-dimension samples (i.e., + a ``init_n_samples^n_inputs`` grid with good marginal coverage). The + FIM is evaluated sequentially at each candidate, then all + ``C(n_candidates, n_exp)`` combinations are scored and the best one + is used as the initial point for the NLP solver. This can + significantly improve solution quality when the problem has multiple + local optima. + + Solver options in LHS worker evaluations: + When ``init_parallel=True``, worker threads construct solver instances + using the same solver name and options as ``self.solver`` (when + available). Therefore, per-solve limits (e.g., iteration/time limits) + configured on the main DoE solver are propagated to candidate FIM + evaluations. + """ + # Check results file name + if results_file is not None: + if not isinstance(results_file, (pathlib.Path, str)): + raise ValueError( + "``results_file`` must be either a Path object or a string." + ) + + # --- Resolve n_exp and determine operating mode --- + n_list = len(self.experiment_list) + if n_list > 1: + # User-initialized mode: experiment list already contains all + # pre-initialized experiment objects. + if n_exp is not None: + raise ValueError( + "``n_exp`` must not be set when the experiment list contains " + f"more than one experiment (got {n_list} experiments in the " + "list). Either pass a single template experiment and set " + "``n_exp``, or pass a fully-initialized list and omit ``n_exp``." + ) + n_exp = n_list + _template_mode = False + else: + # Template mode: single experiment object cloned n_exp times. + if n_exp is None: + n_exp = 1 # default: single-experiment optimization + elif not isinstance(n_exp, int) or n_exp < 1: + raise ValueError( + f"``n_exp`` must be a positive integer, got {n_exp!r}." + ) + _template_mode = True + # --------------------------------------------------- + + # --- Validate initialization arguments --- + if init_method is None: + resolved_init_method = None + else: + try: + resolved_init_method = InitializationMethod(init_method) + except ValueError: + valid = ", ".join(f"'{m.value}'" for m in InitializationMethod) + raise ValueError( + "``init_method`` must be one of [None, " + + valid + + f"], got {init_method!r}." + ) + + if resolved_init_method == InitializationMethod.lhs: + if not _template_mode: + raise ValueError( + "``init_method='lhs'`` is currently supported only in " + "template mode (``len(experiment) == 1``)." + ) + if not scipy_available: + raise ImportError( + "LHS initialization requires scipy. " + "Please install scipy to use init_method='lhs'." + ) + if not isinstance(init_n_samples, int) or init_n_samples < 1: + raise ValueError( + "``init_n_samples`` must be a positive integer, " + f"got {init_n_samples!r}." + ) + if init_seed is not None and not isinstance(init_seed, int): + raise ValueError( + "``init_seed`` must be None or an integer, " f"got {init_seed!r}." + ) + if not isinstance(init_parallel, bool): + raise ValueError( + f"``init_parallel`` must be a bool, got {init_parallel!r}." + ) + if not isinstance(init_combo_parallel, bool): + raise ValueError( + "``init_combo_parallel`` must be a bool, " + f"got {init_combo_parallel!r}." + ) + if init_n_workers is not None and ( + not isinstance(init_n_workers, int) or init_n_workers < 1 + ): + raise ValueError( + "``init_n_workers`` must be None or a positive integer, " + f"got {init_n_workers!r}." + ) + if not isinstance(init_combo_chunk_size, int) or init_combo_chunk_size < 1: + raise ValueError( + "``init_combo_chunk_size`` must be a positive integer, " + f"got {init_combo_chunk_size!r}." + ) + if ( + not isinstance(init_combo_parallel_threshold, int) + or init_combo_parallel_threshold < 1 + ): + raise ValueError( + "``init_combo_parallel_threshold`` must be a positive integer, " + f"got {init_combo_parallel_threshold!r}." + ) + if init_max_wall_clock_time is not None and ( + not isinstance(init_max_wall_clock_time, (int, float)) + or not math.isfinite(init_max_wall_clock_time) + or init_max_wall_clock_time <= 0 + ): + raise ValueError( + "``init_max_wall_clock_time`` must be None or a positive number, " + f"got {init_max_wall_clock_time!r}." + ) + if init_solver is not None and not hasattr(init_solver, "solve"): + raise ValueError( + "``init_solver`` must be None or a solver object with a 'solve' method." + ) + # ----------------------------------------- + primary_solver = self.solver + resolved_init_solver = primary_solver if init_solver is None else init_solver + init_solver_name = getattr( + resolved_init_solver, "name", str(resolved_init_solver) + ) + lhs_init_diagnostics = None + lhs_initialization_time = 0.0 + + # Start timer + sp_timer = TicTocTimer() + sp_timer.tic(msg=None) + self.logger.info( + f"Beginning multi-experiment optimization with {n_exp} experiments." + ) + # Rebuild the multi-experiment model from a clean base each call. + self.model = pyo.ConcreteModel() + + n_param_scenarios = 1 # currently single-scenario optimization + # Use an immutable tuple since weights are not intended to be modified + self.scenario_weights = (1.0,) # Single scenario, weight = 1 + # Add parameter scenario blocks to the model + self.model.param_scenario_blocks = pyo.Block(range(n_param_scenarios)) + symmetry_breaking_info = { + "enabled": n_exp > 1, + "variable": None, + "source": None, + } + diagnostics_warnings = [] + # Route all pre-final solves through the initialization solver, then + # restore the primary solver for the final optimization solve. + self.solver = resolved_init_solver + try: + # Add experiment(s) for each scenario + # TODO: Add s_prev = 0 to handle parameter scenarios + for s in range(n_param_scenarios): + scenario_block = self.model.param_scenario_blocks[s] + scenario_block.exp_blocks = pyo.Block(range(n_exp)) + reference_param_names = None + reference_param_values = None + for k in range(n_exp): + # Generate FIM and Sensitivity expressions for each experiment. + # In template mode all experiments share the single template + # (experiment_index=0); in user-initialized mode each experiment + # maps to its own entry in experiment_list (experiment_index=k). + self.create_doe_model( + model=scenario_block.exp_blocks[k], + experiment_index=0 if _template_mode else k, + _for_multi_experiment=True, # Skip creating L matrix per experiment + ) + if reference_param_names is None: + reference_fd_block = scenario_block.exp_blocks[ + k + ].fd_scenario_blocks[0] + reference_param_names = [ + str(pyo.ComponentUID(param, context=reference_fd_block)) + for param in reference_fd_block.unknown_parameters + ] + reference_param_values = [ + float(value) + for value in reference_fd_block.unknown_parameters.values() + ] + else: + # Multi-experiment aggregation assumes every experiment FIM is + # computed at the same nominal theta. Validate that user- + # initialized experiments share both the unknown-parameter + # labels and their nominal values instead of silently + # overwriting one experiment with another. + current_fd_block = scenario_block.exp_blocks[ + k + ].fd_scenario_blocks[0] + current_param_names = [ + str(pyo.ComponentUID(param, context=current_fd_block)) + for param in current_fd_block.unknown_parameters + ] + # The sensitivity scenarios and resulting FIM bookkeeping both + # depend on the theta ordering used to build each experiment + # block. Require the same parameter order across experiments so + # the aggregated multi-experiment FIM is formed without + # ambiguity or user confusion. + if current_param_names != reference_param_names: + raise ValueError( + "All experiments passed to optimize_experiments() " + "must define the same unknown_parameters in the same " + "order. " + f"Experiment 0 uses {reference_param_names}, while " + f"experiment {k} uses {current_param_names}." + ) + + current_param_values = [ + float(value) + for value in current_fd_block.unknown_parameters.values() + ] + for name, ref_val, cur_val in zip( + reference_param_names, + reference_param_values, + current_param_values, + ): + if not math.isclose( + ref_val, cur_val, rel_tol=1e-12, abs_tol=1e-12 + ): + raise ValueError( + "All experiments passed to optimize_experiments() " + "must use the same nominal values for " + "unknown_parameters. " + f"Parameter '{name}' has value {ref_val} in " + f"experiment 0 and {cur_val} in experiment {k}." + ) + + # Add symmetry breaking constraints to prevent equivalent permutations for + # multiple experiments + if n_exp > 1: + # Check if user provided a symmetry breaking variable via Suffix + # Use first scenario since variable names are the same across all scenarios + first_exp_block = ( + self.model.param_scenario_blocks[0] + .exp_blocks[0] + .fd_scenario_blocks[0] + ) + + # Determine symmetry breaking variable + if ( + hasattr(first_exp_block, 'sym_break_cons') + and len(first_exp_block.sym_break_cons) > 0 + ): + # User provided symmetry breaking variable(s) + sym_break_var_list = list(first_exp_block.sym_break_cons.keys()) + + if len(sym_break_var_list) > 1: + warning_msg = ( + f"Multiple variables marked in sym_break_cons. " + f"Using {sym_break_var_list[0].local_name} for symmetry breaking." + ) + self.logger.warning(warning_msg) + diagnostics_warnings.append(warning_msg) + + sym_break_var = sym_break_var_list[0] + if not any( + sym_break_var is inp + for inp in first_exp_block.experiment_inputs.keys() + ): + raise ValueError( + "Variable selected in ``sym_break_cons`` must also be an " + "experiment input variable. " + f"Got non-input variable '{sym_break_var.local_name}'." + ) + symmetry_breaking_info["variable"] = sym_break_var.local_name + symmetry_breaking_info["source"] = "user" + self.logger.info( + f"Using user-specified variable '{sym_break_var.local_name}' for symmetry breaking." + ) + else: + # Use first experiment input as default symmetry breaking variable + sym_break_var = next(iter(first_exp_block.experiment_inputs)) + symmetry_breaking_info["variable"] = sym_break_var.local_name + symmetry_breaking_info["source"] = "auto" + self.logger.warning( + "No symmetry breaking variable specified. Automatically using the first " + f"experiment input '{sym_break_var.local_name}' for ordering constraints. " + "To specify a different variable, add: " + "m.sym_break_cons = pyo.Suffix(direction=pyo.Suffix.LOCAL); " + "m.sym_break_cons[m.your_variable] = None" + ) + diagnostics_warnings.append( + f"No symmetry breaking variable specified. Automatically using " + f"'{sym_break_var.local_name}'." + ) + + # Add constraints for each scenario + for s in range(n_param_scenarios): + for k in range(1, n_exp): + # Get the variable from experiment k-1 + var_prev = pyo.ComponentUID( + sym_break_var, context=first_exp_block + ).find_component_on( + self.model.param_scenario_blocks[s] + .exp_blocks[k - 1] + .fd_scenario_blocks[0] + ) + + # Get the variable from experiment k + var_curr = pyo.ComponentUID( + sym_break_var, context=first_exp_block + ).find_component_on( + self.model.param_scenario_blocks[s] + .exp_blocks[k] + .fd_scenario_blocks[0] + ) + if var_prev is None or var_curr is None: + raise RuntimeError( + "Failed to map symmetry breaking variable " + f"'{sym_break_var.local_name}' onto scenario {s}, " + f"experiment pair ({k - 1}, {k}). Ensure the variable " + "exists on all experiment blocks with compatible labels." + ) + + # Add symmetry breaking constraint + con_name = f"symmetry_breaking_s{s}_exp{k}" + self.model.param_scenario_blocks[s].add_component( + con_name, pyo.Constraint(expr=var_prev <= var_curr) + ) + + self.logger.info( + f"Added {n_exp - 1} symmetry breaking constraints for scenario {s} " + f"using variable: {sym_break_var.local_name}" + ) + + # Create aggregated objective for multi-experiment optimization + self.create_multi_experiment_objective_function(self.model) + + # Track time required to build the DoE model + build_time = sp_timer.toc(msg=None) + self.logger.info( + "Successfully built the multi-experiment DoE model.\nBuild time: %0.1f seconds" + % build_time + ) + + # --- Apply experiment initialization (if requested) --- + # This must be done AFTER the model is built but BEFORE the square solve + # so that the solver uses the correct starting design. + if resolved_init_method == InitializationMethod.lhs: + lhs_timer = TicTocTimer() + lhs_timer.tic(msg=None) + self.logger.info( + f"Applying LHS initialization with {init_n_samples} samples per " + f"experiment-input dimension..." + ) + best_initial_points, lhs_init_diagnostics = ( + self._lhs_initialize_experiments( + lhs_n_samples=init_n_samples, + lhs_seed=init_seed, + n_exp=n_exp, + lhs_parallel=init_parallel, + lhs_combo_parallel=init_combo_parallel, + lhs_n_workers=init_n_workers, + lhs_combo_chunk_size=init_combo_chunk_size, + lhs_combo_parallel_threshold=init_combo_parallel_threshold, + lhs_max_wall_clock_time=init_max_wall_clock_time, + ) + ) + self.logger.info( + "Setting LHS best-found initial design in the optimization model..." + ) + for s in range(n_param_scenarios): + for k in range(n_exp): + exp_input_vars = self._get_experiment_input_vars( + self.model.param_scenario_blocks[s].exp_blocks[k] + ) + for var, val in zip(exp_input_vars, best_initial_points[k]): + var.set_value(val) + lhs_initialization_time = lhs_timer.toc(msg=None) + + # ------------------------------------------------------ + # Reset delta timing so initialization_time measures only square solve. + sp_timer.tic(msg=None) + + # Solve the square problem first to initialize the FIM and sensitivity constraints + # Deactivate objective expression and objective constraints + self.model.objective.deactivate() + # Deactivate obj_cons for each scenario (holds Cholesky/determinant/trace constraints) + for s in range(n_param_scenarios): + if hasattr(self.model.param_scenario_blocks[s], "obj_cons"): + self.model.param_scenario_blocks[s].obj_cons.deactivate() + + # Fix the design variables across all scenarios and experiments + self._set_experiment_inputs_fixed( + n_param_scenarios=n_param_scenarios, n_exp=n_exp, fixed=True + ) + + # Create and solve dummy objective for initialization + self.model.dummy_obj = pyo.Objective(expr=0, sense=pyo.minimize) + self.solver.solve(self.model, tee=self.tee) + + # Track time to initialize the DoE model + initialization_time = sp_timer.toc(msg=None) + self.logger.info( + ( + "Successfully initialized the multi-experiment DoE model." + "\nInitialization time: %0.1f seconds" % initialization_time + ) + ) + + # Deactivate dummy objective + self.model.dummy_obj.deactivate() + self.model.del_component("dummy_obj") + + # Reactivate objective, obj_cons, and unfix experimental design decisions + self._set_experiment_inputs_fixed( + n_param_scenarios=n_param_scenarios, n_exp=n_exp, fixed=False + ) + self.model.objective.activate() + for s in range(n_param_scenarios): + if hasattr(self.model.param_scenario_blocks[s], "obj_cons"): + self.model.param_scenario_blocks[s].obj_cons.activate() + + # Initialize scenario-level quantities from the square-solve FIM. + # For the algebraic objective path this means Cholesky/determinant + # variables. For the grey box path we explicitly seed the external + # block inputs/outputs from the aggregated scenario FIM before the + # final solve switches to the grey_box_solver. + parameter_names = ( + self.model.param_scenario_blocks[0].exp_blocks[0].parameter_names + ) + + for s in range(n_param_scenarios): + scenario = self.model.param_scenario_blocks[s] + # Update total_fim values from solved individual experiment FIMs + # Individual experiment FIMs don't include prior_FIM in multi-experiment mode, + # so we add it once here to the aggregated total + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + # When only_compute_fim_lower=True, only the lower triangle is computed + # Upper triangle elements are fixed to 0, so only aggregate lower triangle + if self.only_compute_fim_lower and i < j: + continue + + fim_sum = sum( + pyo.value(scenario.exp_blocks[k].fim[p, q]) or 0 + for k in range(n_exp) + ) + scenario.total_fim[p, q].set_value( + fim_sum + self.prior_FIM[i, j] + ) + + # Build scenario total FIM once and reuse for all objective initializations. + total_fim_vals = [ + pyo.value(scenario.total_fim[p, q]) + for p in parameter_names + for q in parameter_names + ] + total_fim_np = np.array(total_fim_vals).reshape( + (len(parameter_names), len(parameter_names)) + ) + if self.only_compute_fim_lower: + total_fim_np = self._symmetrize_lower_tri(total_fim_np) + obj_cons = getattr(scenario, "obj_cons", None) + + if self.use_grey_box: + self._initialize_grey_box_block( + obj_cons.egb_fim_block, total_fim_np, parameter_names + ) + # Initialize scenario-level variables based on total_fim + elif obj_cons is not None and hasattr(obj_cons, "L"): + # Compute Cholesky factorization + # Check positive definiteness and add jitter if needed + min_eig = np.min(np.real(np.linalg.eigvals(total_fim_np))) + if min_eig < _SMALL_TOLERANCE_DEFINITENESS: + jitter = np.min( + [ + -min_eig + _SMALL_TOLERANCE_DEFINITENESS, + _SMALL_TOLERANCE_DEFINITENESS, + ] + ) + else: + jitter = 0 + + fim_jittered = total_fim_np + jitter * np.eye(len(parameter_names)) + + # Compute Cholesky decomposition + L_vals = np.linalg.cholesky(fim_jittered) + + # Initialize L values + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + obj_cons.L[p, q].set_value(L_vals[i, j]) + + # If trace objective, also initialize L_inv, fim_inv, and cov_trace + if hasattr(obj_cons, "L_inv"): + L_inv_vals = np.linalg.inv(L_vals) + + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + if i >= j: + obj_cons.L_inv[p, q].set_value(L_inv_vals[i, j]) + else: + obj_cons.L_inv[p, q].set_value(0.0) + + # Initialize fim_inv + if hasattr(obj_cons, "fim_inv"): + fim_inv_np = np.linalg.inv(fim_jittered) + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + obj_cons.fim_inv[p, q].set_value(fim_inv_np[i, j]) + + # Initialize cov_trace + if hasattr(obj_cons, "cov_trace"): + initial_cov_trace = np.sum(L_inv_vals**2) + obj_cons.cov_trace.set_value(initial_cov_trace) + + if obj_cons is not None and hasattr(obj_cons, "determinant"): + # Initialize determinant + obj_cons.determinant.set_value(np.linalg.det(total_fim_np)) + + if obj_cons is not None and hasattr(obj_cons, "pseudo_trace"): + # Initialize pseudo_trace + pseudo_trace_val = float(np.trace(total_fim_np)) + obj_cons.pseudo_trace.set_value(pseudo_trace_val) + finally: + self.solver = primary_solver + + # Initialization uses the regular/init solver because it operates on the + # algebraic square model. The final NLP solve switches to the grey box + # solver only after the external block has been fully seeded above. + final_solver = self.grey_box_solver if self.use_grey_box else self.solver + final_solver_name = getattr(final_solver, "name", str(final_solver)) + + # Solve the full model + res = final_solver.solve( + self.model, tee=self.grey_box_tee if self.use_grey_box else self.tee + ) + + # Track time used to solve the DoE model + solve_time = sp_timer.toc(msg=None) + + self.logger.info( + ( + "Successfully optimized multi-experiment design." + "\nSolve time: %0.1f seconds" % solve_time + ) + ) + self.logger.info( + "Total time for build, initialization, and solve: %0.1f seconds" + % (build_time + initialization_time + solve_time) + ) + + # Collect results + solver_status = str(res.solver.status) + termination_condition = str(res.solver.termination_condition) + if isinstance(res.solver.message, str): + results_message = res.solver.message + elif isinstance(res.solver.message, bytes): + results_message = res.solver.message.decode("utf-8") + else: + results_message = ( + str(res.solver.message) if res.solver.message is not None else "" + ) + + def _safe_metric(metric_name, compute_fn, scenario_index): + try: + val = float(compute_fn()) + return val if np.isfinite(val) else float("nan") + except Exception as exc: + self.logger.warning( + f"Scenario {scenario_index}: failed to compute {metric_name}: {exc}. " + "Setting metric to NaN." + ) + return float("nan") + + # Store results for each parameter scenario using a single structured + # payload. The outer ``solution.param_scenarios`` list is always present + # so the API shape stays the same for single- and multi-scenario cases. + param_scenarios = [] + for s in range(n_param_scenarios): + scenario = self.model.param_scenario_blocks[s] + + # Get aggregated FIM for this scenario + total_fim_vals = [ + pyo.value(scenario.total_fim[p, q]) + for p in parameter_names + for q in parameter_names + ] + total_fim_np = np.array(total_fim_vals).reshape( + (len(parameter_names), len(parameter_names)) + ) + + # Complete FIM if only computing lower triangle + if self.only_compute_fim_lower: + total_fim_np = self._symmetrize_lower_tri(total_fim_np) + + # Statistics on the aggregated FIM (consistent with run_doe), guarded + # against singular/indefinite matrices and numerical failures. + log10_a_opt = _safe_metric( + "log10 A-opt", + lambda fim=total_fim_np: np.log10(np.trace(np.linalg.inv(fim))), + s, + ) + log10_pseudo_a_opt = _safe_metric( + "log10 pseudo A-opt", + lambda fim=total_fim_np: np.log10(np.trace(fim)), + s, + ) + log10_d_opt = _safe_metric( + "log10 D-opt", lambda fim=total_fim_np: np.log10(np.linalg.det(fim)), s + ) + log10_e_opt = _safe_metric( + "log10 E-opt", + lambda fim=total_fim_np: np.log10(min(np.linalg.eigvalsh(fim))), + s, + ) + log10_me_opt = _safe_metric( + "log10 ME-opt", + lambda fim=total_fim_np: np.log10(np.linalg.cond(fim)), + s, + ) + # Scenario-level results capture the aggregate design quality and the + # common parameter values used by all experiments in the scenario. + scenario_structured = { + "param_scenario_id": s, + "param_scenario_weight": float(self.scenario_weights[s]), + "total_fim": total_fim_np.tolist(), + "parameter_values": self.get_unknown_parameter_values( + model=scenario.exp_blocks[0] + ), + "quality_metrics": { + "log10_a_opt": log10_a_opt, + "log10_pseudo_a_opt": log10_pseudo_a_opt, + "log10_d_opt": log10_d_opt, + "log10_e_opt": log10_e_opt, + "log10_me_opt": log10_me_opt, + }, + "experiments": [], + } + for k in range(n_exp): + exp_block = scenario.exp_blocks[k] + # Each experiment entry carries only quantities that differ from + # one experiment block to the next. Measurement-error data is + # reported once at the top level as shared problem metadata. + experiment_structured = { + "exp_id": k, + "design": self.get_experiment_input_values(model=exp_block), + "outputs": self.get_experiment_output_values(model=exp_block), + "fim": self.get_FIM(model=exp_block), + "sensitivity": None, + } + if hasattr(exp_block, "sensitivity_jacobian"): + experiment_structured["sensitivity"] = self.get_sensitivity_matrix( + model=exp_block + ) + scenario_structured["experiments"].append(experiment_structured) + + param_scenarios.append(scenario_structured) + + # Store labels once because they describe the axes of every numeric list + # and matrix in the payload. + first_exp_block_fd = ( + self.model.param_scenario_blocks[0].exp_blocks[0].fd_scenario_blocks[0] + ) + design_variable_labels = [ + str(pyo.ComponentUID(comp, context=first_exp_block_fd)) + for comp in first_exp_block_fd.experiment_inputs + ] + output_labels = [ + str(pyo.ComponentUID(comp, context=first_exp_block_fd)) + for comp in first_exp_block_fd.experiment_outputs + ] + parameter_labels = [ + str(pyo.ComponentUID(comp, context=first_exp_block_fd)) + for comp in first_exp_block_fd.unknown_parameters + ] + measurement_error_labels = [ + str(pyo.ComponentUID(comp, context=first_exp_block_fd)) + for comp in first_exp_block_fd.measurement_error + ] + measurement_error_values = self.get_measurement_error_values( + model=self.model.param_scenario_blocks[0].exp_blocks[0] + ) + total_time = ( + build_time + lhs_initialization_time + initialization_time + solve_time + ) + initialization_method = ( + resolved_init_method.value if resolved_init_method is not None else "none" + ) + lhs_details = lhs_init_diagnostics or {} + + self.results = { + "problem": { + # Each parameter scenario corresponds to one parameter vector in + # the solve, so this count matches the length of + # ``solution["param_scenarios"]``. + "number_of_param_scenarios": n_param_scenarios, + "number_of_experiments_per_scenario": n_exp, + "used_template_experiment": _template_mode, + "finite_difference_scheme": self._enum_label(self.fd_formula), + "finite_difference_step": self.step, + "scaled_nominal_parameters": self.scale_nominal_param_value, + "prior_fim": self.prior_FIM.tolist(), + "measurement_error_values": measurement_error_values, + "design_variables": design_variable_labels, + "outputs": output_labels, + "parameters": parameter_labels, + "measurement_errors": measurement_error_labels, + }, + "initialization": { + "method": initialization_method, + "solver": init_solver_name, + "samples_per_design_variable": ( + init_n_samples + if resolved_init_method == InitializationMethod.lhs + else None + ), + "random_seed": ( + init_seed + if resolved_init_method == InitializationMethod.lhs + else None + ), + "parallel_candidate_fim_evaluation": ( + init_parallel + if resolved_init_method == InitializationMethod.lhs + else None + ), + "parallel_combination_scoring": ( + init_combo_parallel + if resolved_init_method == InitializationMethod.lhs + else None + ), + "workers": ( + init_n_workers + if resolved_init_method == InitializationMethod.lhs + else None + ), + "combination_chunk_size": ( + init_combo_chunk_size + if resolved_init_method == InitializationMethod.lhs + else None + ), + "parallel_combination_threshold": ( + init_combo_parallel_threshold + if resolved_init_method == InitializationMethod.lhs + else None + ), + "max_time_s": ( + init_max_wall_clock_time + if resolved_init_method == InitializationMethod.lhs + else None + ), + "candidate_fim_evaluation_mode": lhs_details.get("candidate_fim_mode"), + "combination_scoring_mode": lhs_details.get("combo_mode"), + "number_of_candidate_designs": lhs_details.get("n_candidates"), + "number_of_design_combinations": lhs_details.get("n_combinations"), + "sampling_time_s": lhs_details.get("elapsed_sampling_s"), + "candidate_fim_evaluation_time_s": lhs_details.get( + "elapsed_fim_eval_s" + ), + "combination_scoring_time_s": lhs_details.get( + "elapsed_combo_scoring_s" + ), + "time_s": lhs_details.get("elapsed_total_s"), + "timed_out": lhs_details.get("timed_out"), + "selected_initial_designs": ( + best_initial_points + if resolved_init_method == InitializationMethod.lhs + else None + ), + "best_initial_objective_value": lhs_details.get("best_obj"), + "best_initial_objective_value_log10": lhs_details.get("best_obj_log10"), + }, + "experiment_ordering": { + "design_variable": symmetry_breaking_info["variable"], + "chosen_by": symmetry_breaking_info["source"], + }, + "optimization_solve": { + "solver": final_solver_name, + "status": solver_status, + "termination_condition": termination_condition, + "message": results_message, + }, + "timing": { + "build_time_s": build_time, + "initialization_time_s": initialization_time, + "lhs_initialization_time_s": lhs_initialization_time, + "optimization_solve_time_s": solve_time, + "total_time_s": total_time, + }, + "warnings": diagnostics_warnings, + "solution": { + "objective": self._enum_label(self.objective_option), + "param_scenarios": param_scenarios, + }, + } + + # Save results to file if requested + if results_file is not None: + with open(results_file, "w") as file: + json.dump(self.results, file, indent=2, cls=_DoEResultsJSONEncoder) + + # LHS-initialization helpers + def _get_experiment_input_vars(self, exp_block): + """ + Return the experiment-input Pyomo variable objects for an experiment + block, abstracting over the specific sensitivity-computation structure + (FD, AD, etc.). + + When the block exposes ``experiment_inputs`` directly (e.g. in a future + automatic-differentiation path), those are used. Otherwise the method + falls back to the FD structure (``exp_block.fd_scenario_blocks[0]``). + + Parameters + ---------- + exp_block : Pyomo Block + An ``exp_blocks[k]`` sub-block of the multi-experiment model. + + Returns + ------- + list + Ordered list of Pyomo :class:`Var` objects corresponding to the + experiment inputs. + """ + if hasattr(exp_block, "experiment_inputs"): + return list(exp_block.experiment_inputs.keys()) + # FD structure: inputs live on the base finite-difference scenario block + return list(exp_block.fd_scenario_blocks[0].experiment_inputs.keys()) + + def _set_experiment_inputs_fixed(self, n_param_scenarios, n_exp, fixed): + """Fix or unfix experiment inputs across all scenarios and experiments.""" + for s in range(n_param_scenarios): + for k in range(n_exp): + for comp in ( + self.model.param_scenario_blocks[s] + .exp_blocks[k] + .fd_scenario_blocks[0] + .experiment_inputs + ): + if fixed: + comp.fix() + else: + comp.unfix() + + @staticmethod + def _evaluate_objective_for_option(fim_matrix, objective_option): + _bad = ( + -np.inf + if objective_option in DesignOfExperiments._MAXIMIZE_OBJECTIVES + else np.inf + ) + + try: + if objective_option == ObjectiveLib.determinant: + return float(np.linalg.det(fim_matrix)) + elif objective_option == ObjectiveLib.pseudo_trace: + return float(np.trace(fim_matrix)) + elif objective_option == ObjectiveLib.trace: + return float(np.trace(np.linalg.inv(fim_matrix))) + elif objective_option == ObjectiveLib.minimum_eigenvalue: + return float(np.min(np.linalg.eigvalsh(fim_matrix))) + elif objective_option == ObjectiveLib.condition_number: + return float(np.linalg.cond(fim_matrix)) + else: # zero or unknown + return 0.0 + except (np.linalg.LinAlgError, ValueError): + return _bad + + def _evaluate_objective_from_fim(self, fim_matrix): + """ + Compute the scalar DoE objective from a numpy FIM matrix. + + Parameters + ---------- + fim_matrix : np.ndarray + Square FIM to score. + + Returns + ------- + float + Objective value. For maximisation objectives (``determinant``, + ``pseudo_trace``, ``minimum_eigenvalue``) a larger value is better. + For minimisation objectives (``trace`` / A-optimality and + ``condition_number`` / ME-optimality) a smaller value is better. + """ + return self._evaluate_objective_for_option(fim_matrix, self.objective_option) + + @staticmethod + def _symmetrize_lower_tri(mat): + """Mirror lower-triangle FIM entries to the upper triangle.""" + return mat + mat.T - np.diag(np.diag(mat)) + + @staticmethod + def _make_cholesky_rule(fim_expr, L_expr, parameter_names): + """ + Return a constraint rule that enforces ``fim_expr = L_expr * L_expr^T`` + on the lower-triangular portion defined by ``parameter_names``. + + The produced rule follows the Pyomo signature ``rule(block, p, q)`` + but does **not** actually use `block` in its body; the two matrix + expressions are captured from the enclosing scope. + + Parameters + ---------- + fim_expr : Var-like + Indexed by ``(p, q)``; usually ``model.fim`` or + ``scenario.total_fim``. + L_expr : Var-like + Indexed by ``(p, q)``; the corresponding lower-triangular + Cholesky factors. + parameter_names : Set + Pyomo Set listing the parameter indices in order. + """ + + def rule(_b, p, q): + p_idx = list(parameter_names).index(p) + q_idx = list(parameter_names).index(q) + if p_idx >= q_idx: + return fim_expr[p, q] == sum( + L_expr[p, parameter_names.at(k + 1)] + * L_expr[q, parameter_names.at(k + 1)] + for k in range(q_idx + 1) + ) + else: + return pyo.Constraint.Skip + + return rule + + @staticmethod + def _make_cholesky_inv_rule(fim_inv_expr, L_inv_expr, parameter_names): + """ + Return a rule that enforces ``fim_inv_expr = L_inv_expr^T * L_inv_expr`` + over the lower-triangular index region. + """ + + def rule(_b, p, q): + p_idx = list(parameter_names).index(p) + q_idx = list(parameter_names).index(q) + if p_idx >= q_idx: + return fim_inv_expr[p, q] == sum( + L_inv_expr[parameter_names.at(k + 1), p] + * L_inv_expr[parameter_names.at(k + 1), q] + for k in range(p_idx, len(parameter_names)) + ) + return pyo.Constraint.Skip + + return rule + + @staticmethod + def _make_cholesky_LLinv_rule(L_expr, L_inv_expr, parameter_names): + """ + Return a rule that enforces ``L_expr * L_inv_expr = I`` over the + lower-triangular index region. + """ + + def rule(_b, p, q): + p_idx = list(parameter_names).index(p) + q_idx = list(parameter_names).index(q) + if p_idx < q_idx: + return pyo.Constraint.Skip + target = 1 if p_idx == q_idx else 0 + return ( + sum( + L_expr[p, parameter_names.at(k + 1)] + * L_inv_expr[parameter_names.at(k + 1), q] + for k in range(len(parameter_names)) + ) + == target + ) + + return rule + + def _compute_fim_at_point_no_prior(self, experiment_index, input_values): + """ + Compute the FIM (without the prior FIM contribution) for the given + experiment at the specified experiment-input values using the + sequential finite-difference method. + + Parameters + ---------- + experiment_index : int + Index of the experiment in ``self.experiment_list`` to evaluate. + input_values : list + Numeric values for each experiment input variable (same order as + ``model.experiment_inputs``). + + Returns + ------- + np.ndarray + ``(n_params, n_params)`` FIM matrix, **excluding** the prior. + A zero matrix is returned on solver failure (with a warning). + """ + # Get a fresh labeled model for this experiment + model = ( + self.experiment_list[experiment_index] + .get_labeled_model(**self.get_labeled_model_args) + .clone() + ) + self.check_model_labels(model=model) + n_params = len(model.unknown_parameters) + + # Override experiment input values + update_model_from_suffix( + suffix_obj=model.experiment_inputs, values=input_values + ) + + # Temporarily zero the prior so that seq_FIM = Q^T * 𝚺^{-1} * Q only + saved_prior = self.prior_FIM + self.prior_FIM = np.zeros((n_params, n_params)) + + try: + self._sequential_FIM(model=model) + fim = self.seq_FIM.copy() + except Exception as exc: + self.logger.warning( + f"FIM evaluation failed at point {input_values}: {exc}. " + "Using zero matrix as fallback." + ) + fim = np.zeros((n_params, n_params)) + finally: + self.prior_FIM = saved_prior + + return fim + + def _lhs_initialize_experiments( + self, + lhs_n_samples, + lhs_seed, + n_exp, + lhs_parallel=False, + lhs_combo_parallel=False, + lhs_n_workers=None, + lhs_combo_chunk_size=5000, + lhs_combo_parallel_threshold=20000, + lhs_max_wall_clock_time=None, + ): + """ + Use per-dimension Latin Hypercube Sampling to identify a good initial + experiment design for ``optimize_experiments``. + """ + # Use a monotonic wall-clock for progress estimates and deadline checks + # in both serial and threaded LHS evaluation loops. + t_start = time.perf_counter() + + # 1. Get experiment-input bounds from the already-built model + first_exp_block = self.model.param_scenario_blocks[0].exp_blocks[0] + exp_input_vars = self._get_experiment_input_vars(first_exp_block) + n_inputs = len(exp_input_vars) + + missing = [v.name for v in exp_input_vars if v.lb is None or v.ub is None] + if missing: + raise ValueError( + "LHS initialization requires explicit lower and upper bounds on " + "all experiment input variables. The following variables are " + f"missing bounds: {missing}. " + "Set bounds in your experiment input variables before " + "calling ``optimize_experiments`` with " + "``init_method='lhs'``." + ) + + lb_vals = np.array([v.lb for v in exp_input_vars]) + ub_vals = np.array([v.ub for v in exp_input_vars]) + + # 2. Generate per-dimension 1-D LHS samples and take Cartesian product + # Split the user seed into per-dimension seeds so each 1-D LHS draw + # is independent while remaining reproducible for a fixed lhs_seed. + rng = np.random.default_rng(lhs_seed) + per_dim_samples = [] + for i in range(n_inputs): + dim_seed = int(rng.integers(0, 2**31)) + sampler = LatinHypercube(d=1, seed=dim_seed) + s_unit = sampler.random(n=lhs_n_samples).flatten() + s_scaled = lb_vals[i] + s_unit * (ub_vals[i] - lb_vals[i]) + per_dim_samples.append(s_scaled.tolist()) + + candidate_points = tuple(product(*per_dim_samples)) + t_after_sampling = time.perf_counter() + n_candidates = len(candidate_points) + + if n_candidates > 10_000: + warnings.warn( + f"LHS initialization generated {n_candidates:,} candidate " + f"experiment designs (lhs_n_samples={lhs_n_samples}, " + f"n_inputs={n_inputs}). Evaluating FIM at all candidates may " + "take a long time. Consider reducing ``lhs_n_samples``.", + UserWarning, + stacklevel=2, + ) + + if hasattr(first_exp_block, "fd_scenario_blocks"): + n_scenarios_per_candidate = len(list(first_exp_block.fd_scenario_blocks)) + else: + n_scenarios_per_candidate = 1 + # Change the following block if we add support for LHS initialization with + # non-FD structures (e.g. AD) + if ( + not hasattr(first_exp_block, "fd_scenario_blocks") + or len(first_exp_block.fd_scenario_blocks) == 0 + ): + raise RuntimeError( + "_lhs_initialize_experiments requires finite-difference scenario " + "blocks on the experiment model. Ensure optimize_experiments is " + "using the sequential FIM path." + ) + + resolved_workers = ( + lhs_n_workers + if lhs_n_workers is not None + else max(1, min(os.cpu_count() or 1, 8)) + ) + use_parallel_fim = lhs_parallel and resolved_workers > 1 + fim_eval_mode = "parallel" if use_parallel_fim else "serial" + self.logger.info( + f"LHS: evaluating FIM at {n_candidates} candidate designs " + f"({n_candidates * n_scenarios_per_candidate} solver calls expected; " + f"{fim_eval_mode} candidate FIM mode)." + ) + + # Track worker DoE construction failures to avoid repeated logging of the same issue + _solver_fallback_lock = threading.Lock() + _solver_fallback_logged = False + + def _make_worker_solver(): + nonlocal _solver_fallback_logged + solver_name = getattr(self.solver, "name", None) + if solver_name is None: + with _solver_fallback_lock: + if not _solver_fallback_logged: + self.logger.debug( + "LHS parallel: solver has no 'name' attribute; worker DoE " + "will use default solver settings." + ) + _solver_fallback_logged = True + return None + worker_solver = pyo.SolverFactory(solver_name) + if worker_solver is None: + with _solver_fallback_lock: + if not _solver_fallback_logged: + self.logger.debug( + f"LHS parallel: could not construct solver '{solver_name}'; " + "worker DoE will use default solver settings." + ) + _solver_fallback_logged = True + return None + try: + if hasattr(self.solver, "options") and hasattr( + worker_solver, "options" + ): + worker_solver.options.update(self.solver.options) + except Exception: + pass + return worker_solver + + thread_state = threading.local() + n_params = len(first_exp_block.fd_scenario_blocks[0].unknown_parameters) + + def _compute_candidate_fim(idx_pt): + idx, pt = idx_pt + try: + worker_doe = getattr(thread_state, "doe", None) + if worker_doe is None: + # Retry worker DoE construction on subsequent points even if a + # previous point failed; transient failures should not disable + # the rest of this thread's candidate evaluations. + worker_solver = _make_worker_solver() + worker_doe = DesignOfExperiments( + experiment=self.experiment_list, + fd_formula=self.fd_formula.value, + step=self.step, + objective_option=self.objective_option.value, + use_grey_box_objective=self.use_grey_box, + scale_constant_value=self.scale_constant_value, + scale_nominal_param_value=self.scale_nominal_param_value, + prior_FIM=self.prior_FIM, + jac_initial=self.jac_initial, + fim_initial=self.fim_initial, + L_diagonal_lower_bound=self.L_diagonal_lower_bound, + solver=worker_solver, + grey_box_solver=self.grey_box_solver, + tee=False, + grey_box_tee=False, + get_labeled_model_args=self.get_labeled_model_args, + logger_level=logging.ERROR, + improve_cholesky_roundoff_error=self.improve_cholesky_roundoff_error, + _Cholesky_option=self.Cholesky_option, + _only_compute_fim_lower=self.only_compute_fim_lower, + ) + thread_state.doe = worker_doe + # LHS initialization evaluates candidate points against the + # canonical experiment template (experiment_list[0]). + fim = worker_doe._compute_fim_at_point_no_prior( + experiment_index=0, input_values=list(pt) + ) + return idx, fim + except Exception as exc: + if not getattr(thread_state, "construction_failed", False): + thread_state.construction_failed = True + self.logger.error( + f"LHS: worker DoE construction/evaluation failed on thread " + f"{threading.current_thread().name}: {exc}. " + "Using zero FIM for this candidate and continuing." + ) + return idx, np.zeros((n_params, n_params)) + + # 3. Evaluate FIM at every candidate design + candidate_fims = [None] * n_candidates + if lhs_parallel and not use_parallel_fim: + self.logger.warning( + "LHS candidate FIM parallel evaluation requested with " + "``lhs_parallel=True``, but running serially: " + f"resolved_workers={resolved_workers} <= 1." + ) + timed_out = False + deadline = ( + None + if lhs_max_wall_clock_time is None + else (t_start + lhs_max_wall_clock_time) + ) + if use_parallel_fim: + self.logger.info( + f"LHS: using parallel candidate FIM evaluation with {resolved_workers} workers." + ) + idx_iter = iter(enumerate(candidate_points)) + max_pending = max(1, 2 * resolved_workers) + with _cf.ThreadPoolExecutor(max_workers=resolved_workers) as ex: + pending = set() + n_done = 0 + while True: + while len(pending) < max_pending: + if deadline is not None and time.perf_counter() > deadline: + timed_out = True + break + try: + idx_pt = next(idx_iter) + except StopIteration: + break + pending.add(ex.submit(_compute_candidate_fim, idx_pt)) + + if not pending: + break + + done_now, pending = _cf.wait( + pending, timeout=0.1, return_when=_cf.FIRST_COMPLETED + ) + for fut in done_now: + pt_idx, fim = fut.result() + candidate_fims[pt_idx] = fim + n_done += 1 + if n_done % max(1, n_candidates // 10) == 0: + elapsed = time.perf_counter() - t_start + frac_done = n_done / n_candidates + est_total = elapsed / frac_done if frac_done > 0 else 0 + self.logger.info( + f" LHS FIM eval: {n_done}/{n_candidates} " + f"({elapsed:.1f}s elapsed, ~{est_total:.1f}s total)" + ) + + if timed_out: + for fut in pending: + fut.cancel() + break + else: + for pt_idx, pt in enumerate(candidate_points): + if deadline is not None and time.perf_counter() > deadline: + timed_out = True + break + fim = self._compute_fim_at_point_no_prior( + experiment_index=0, input_values=list(pt) + ) + candidate_fims[pt_idx] = fim + if (pt_idx + 1) % max(1, n_candidates // 10) == 0: + elapsed = time.perf_counter() - t_start + frac_done = (pt_idx + 1) / n_candidates + est_total = elapsed / frac_done if frac_done > 0 else 0 + self.logger.info( + f" LHS FIM eval: {pt_idx + 1}/{n_candidates} " + f"({elapsed:.1f}s elapsed, ~{est_total:.1f}s total)" + ) + + computed_pairs = [ + (pt, fim) + for pt, fim in zip(candidate_points, candidate_fims) + if fim is not None + ] + + # If no candidates were successfully evaluated, use the first candidate with + # a zero FIM to allow downstream code to run without missing data. + # This is an extreme fallback that should only occur if every candidate + # evaluation failed or if the time budget was too small to evaluate + # any candidates; in either case we want to avoid crashing and allow + # the user to get something back (with warnings) rather than nothing. + if not computed_pairs: + timed_out = True + computed_pairs = [(candidate_points[0], np.zeros((n_params, n_params)))] + + # If timeout stops FIM evaluation early, retain only scored candidates so + # downstream combination scoring does not use missing entries. + if len(computed_pairs) < n_candidates: + self.logger.warning( + "LHS candidate FIM evaluation reached time budget " + f"({lhs_max_wall_clock_time}s). Scored {len(computed_pairs)}/{n_candidates} " + "candidates; continuing with best available subset." + ) + if len(computed_pairs) < n_exp: + first_pt, first_fim = computed_pairs[0] + computed_pairs.extend( + (first_pt, first_fim.copy()) + for _ in range(n_exp - len(computed_pairs)) + ) + _pts, _fims = zip(*computed_pairs) + candidate_points = tuple(_pts) + candidate_fims = tuple(_fims) + n_candidates = len(candidate_points) + + t_after_fim = time.perf_counter() + fim_eval_time = t_after_fim - t_after_sampling + self.logger.info(f"LHS: completed FIM evaluations in {fim_eval_time:.1f}s.") + + # 4. Enumerate combinations and score + n_combinations = math.comb(n_candidates, n_exp) + self.logger.info( + f"LHS: scoring {n_combinations:,} combinations of {n_exp} experiments..." + ) + if n_combinations > 100_000: + self.logger.warning( + f"LHS: {n_combinations:,} combinations to evaluate. " + "This may be slow. Consider reducing ``lhs_n_samples``." + ) + + prior = self.prior_FIM.copy() + _obj_option = self.objective_option + is_maximize = _obj_option in self._MAXIMIZE_OBJECTIVES + best_obj = -np.inf if is_maximize else np.inf + best_combo = None + _score_obj = DesignOfExperiments._evaluate_objective_for_option + + def _score_chunk(combo_chunk, deadline_ts): + local_best_obj = -np.inf if is_maximize else np.inf + local_best_combo = None + local_timed_out = False + for combo in combo_chunk: + if deadline_ts is not None and time.perf_counter() > deadline_ts: + local_timed_out = True + break + if n_exp == 2: + i, j = combo + fim_total = prior + candidate_fims[i] + candidate_fims[j] + else: + fim_total = prior.copy() + for idx in combo: + fim_total = fim_total + candidate_fims[idx] + obj_val = _score_obj(fim_total, _obj_option) + if is_maximize: + if obj_val > local_best_obj: + local_best_obj = obj_val + local_best_combo = combo + else: + if obj_val < local_best_obj: + local_best_obj = obj_val + local_best_combo = combo + return local_best_obj, local_best_combo, local_timed_out + + use_parallel_combo = ( + lhs_combo_parallel + and n_combinations >= lhs_combo_parallel_threshold + and resolved_workers > 1 + ) + if lhs_combo_parallel and not use_parallel_combo: + reasons = [] + if n_combinations < lhs_combo_parallel_threshold: + reasons.append( + f"n_combinations={n_combinations} < " + f"lhs_combo_parallel_threshold={lhs_combo_parallel_threshold}" + ) + if resolved_workers <= 1: + reasons.append(f"resolved_workers={resolved_workers} <= 1") + reason_txt = ( + "; ".join(reasons) if reasons else "parallel preconditions not met" + ) + self.logger.warning( + "LHS combination scoring requested with " + "``lhs_combo_parallel=True``, but running serially: " + f"{reason_txt}." + ) + + if use_parallel_combo: + self.logger.info( + f"LHS: using parallel combination scoring with {resolved_workers} workers " + f"(chunk_size={lhs_combo_chunk_size})." + ) + combo_iter = _combinations(range(n_candidates), n_exp) + max_pending = max(1, 2 * resolved_workers) + with _cf.ThreadPoolExecutor(max_workers=resolved_workers) as ex: + pending = set() + while True: + while len(pending) < max_pending: + if deadline is not None and time.perf_counter() > deadline: + timed_out = True + break + chunk = list(_islice(combo_iter, lhs_combo_chunk_size)) + if not chunk: + break + pending.add(ex.submit(_score_chunk, chunk, deadline)) + if not pending: + break + + done, pending = _cf.wait(pending, return_when=_cf.FIRST_COMPLETED) + for fut in done: + local_obj, local_combo, local_timed_out = fut.result() + timed_out = timed_out or local_timed_out + if local_combo is None: + continue + if is_maximize: + if local_obj > best_obj: + best_obj = local_obj + best_combo = local_combo + else: + if local_obj < best_obj: + best_obj = local_obj + best_combo = local_combo + + if deadline is not None and time.perf_counter() > deadline: + timed_out = True + for fut in pending: + fut.cancel() + break + else: + for combo in _combinations(range(n_candidates), n_exp): + if deadline is not None and time.perf_counter() > deadline: + timed_out = True + break + if n_exp == 2: + i, j = combo + fim_total = prior + candidate_fims[i] + candidate_fims[j] + else: + fim_total = prior.copy() + for idx in combo: + fim_total = fim_total + candidate_fims[idx] + obj_val = _score_obj(fim_total, _obj_option) + if is_maximize: + if obj_val > best_obj: + best_obj = obj_val + best_combo = combo + else: + if obj_val < best_obj: + best_obj = obj_val + best_combo = combo + + if timed_out: + self.logger.warning( + f"LHS combination scoring reached time budget " + f"({lhs_max_wall_clock_time}s). Returning best-so-far." + ) + + t_after_combo = time.perf_counter() + combo_time = t_after_combo - t_after_fim + + if best_combo is None: + self.logger.warning( + "LHS combination scoring ended before any combination was scored. " + "Falling back to the first n_exp candidate points." + ) + best_combo = tuple(range(n_exp)) + if n_exp == 2: + i, j = best_combo + fim_total = prior + candidate_fims[i] + candidate_fims[j] + else: + fim_total = prior.copy() + for idx in best_combo: + fim_total = fim_total + candidate_fims[idx] + best_obj = float(_score_obj(fim_total, _obj_option)) + + best_obj_log10 = ( + float(np.log10(best_obj)) + if np.isfinite(best_obj) and best_obj > 0 + else None + ) + self.logger.info( + f"LHS: best {self.objective_option.value} objective = {best_obj:.6g} " + f"(combo scoring took {combo_time:.1f}s)." + ) + + best_initial_points = [list(candidate_points[i]) for i in best_combo] + self.logger.info( + f"LHS initial design: " + + ", ".join(f"exp[{k}]={best_initial_points[k]}" for k in range(n_exp)) + ) + + lhs_diagnostics = { + "candidate_fim_mode": "thread" if use_parallel_fim else "serial", + "combo_mode": "thread" if use_parallel_combo else "serial", + "n_workers": resolved_workers, + "n_candidates": n_candidates, + "n_combinations": n_combinations, + "elapsed_sampling_s": t_after_sampling - t_start, + "elapsed_fim_eval_s": fim_eval_time, + "elapsed_combo_scoring_s": combo_time, + "elapsed_total_s": t_after_combo - t_start, + "timed_out": timed_out, + "time_budget_s": lhs_max_wall_clock_time, + "best_obj": best_obj, + "best_obj_log10": best_obj_log10, + } + return best_initial_points, lhs_diagnostics + # Perform multi-experiment doe (sequential, or ``greedy`` approach) def run_multi_doe_sequential(self, N_exp=1): raise NotImplementedError("Multiple experiment optimization not yet supported.") @@ -556,16 +2304,34 @@ def compute_FIM(self, model=None, method="sequential"): ---------- model: model to compute FIM, default: None, (self.compute_FIM_model) method: string to specify which method should be used - options are ``kaug`` and ``sequential`` + options are ``kaug`` and ``sequential``. When the + ``gradient_method`` is ``pynumero`` or ``kaug``, the + analytic sensitivity path is used regardless of this + argument. + + Notes + ----- + When ``model is None`` and ``experiment_list`` contains multiple + experiments, this method computes each experiment FIM and returns + the aggregate: + + ``total_fim = sum(fim_i for each experiment i) + prior_FIM`` + + where each ``fim_i`` excludes the prior contribution. + Returns ------- computed FIM: 2D numpy array of the FIM """ + aggregate_all_experiments = model is None and len(self.experiment_list) > 1 + if model is None: - self.compute_FIM_model = self.experiment.get_labeled_model( - **self.get_labeled_model_args - ).clone() + self.compute_FIM_model = ( + self.experiment_list[0] + .get_labeled_model(**self.get_labeled_model_args) + .clone() + ) model = self.compute_FIM_model else: # TODO: Add safe naming when a model is passed by the user. @@ -595,22 +2361,168 @@ def compute_FIM(self, model=None, method="sequential"): # TODO: Add a check to see if the model has an objective and deactivate it. # This solve should only be a square solve without any obj function. - if method == "sequential": - self._sequential_FIM(model=model) - self._computed_FIM = self.seq_FIM - elif method == "kaug": - self._kaug_FIM(model=model) - self._computed_FIM = self.kaug_FIM - else: - raise ValueError( - ( - "The method provided, {}, must be either `sequential` " - "or `kaug`".format(method) + def _compute_fim_for_model(eval_model): + if self._gradient_method in [GradientMethod.kaug, GradientMethod.pynumero]: + self._analytic_FIM(model=eval_model) + return np.array(self.analytic_FIM, copy=True) + elif method == "sequential": + self._sequential_FIM(model=eval_model) + return np.array(self.seq_FIM, copy=True) + elif method == "kaug": + self._kaug_FIM(model=eval_model) + return np.array(self.kaug_FIM, copy=True) + else: + raise ValueError( + ( + "The method provided, {}, must be either `sequential` " + "or `kaug`".format(method) + ) ) + + def _unknown_parameter_signature(eval_model): + # Use stable model-local component identifiers and values so we can + # verify all experiments are consistent before aggregating FIMs. + names = [ + str(pyo.ComponentUID(param, context=eval_model)) + for param in eval_model.unknown_parameters + ] + values = np.array( + [ + float(pyo.value(val)) + for val in eval_model.unknown_parameters.values() + ] ) + return names, values + + if aggregate_all_experiments: + saved_prior = self.prior_FIM + self._computed_FIM_by_experiment = [] + # Capture the baseline parameter labels/values from experiment 0. + reference_param_names, reference_param_values = ( + _unknown_parameter_signature(model) + ) + + try: + # Compute each experiment FIM without prior so the aggregate adds + # prior exactly once at the end. + self.prior_FIM = np.zeros(saved_prior.shape) + for idx, exp in enumerate(self.experiment_list): + if idx == 0: + # Reuse the already-built model for experiment 0. + exp_model = model + else: + exp_model = exp.get_labeled_model( + **self.get_labeled_model_args + ).clone() + self.check_model_labels(model=exp_model) + + param_names, param_values = _unknown_parameter_signature(exp_model) + # Reject heterogeneous parameter spaces before solving so + # summed FIMs remain well-defined. + if param_names != reference_param_names: + raise ValueError( + "All experiments in 'experiment_list' must share the same " + "unknown parameter labels and order for compute_FIM " + f"aggregation. Mismatch detected at experiment index {idx}." + ) + if not np.allclose( + param_values, reference_param_values, atol=1e-12, rtol=1e-12 + ): + raise ValueError( + "All experiments in 'experiment_list' must share the same " + "unknown parameter values for compute_FIM aggregation. " + f"Mismatch detected at experiment index {idx}." + ) + + fim_i = _compute_fim_for_model(exp_model) + self._computed_FIM_by_experiment.append(fim_i) + finally: + self.prior_FIM = saved_prior + + # Aggregate all experiment FIMs and add the saved prior once. + total_fim = np.zeros(saved_prior.shape) + for fim_i in self._computed_FIM_by_experiment: + total_fim = total_fim + fim_i + self._computed_FIM = total_fim + saved_prior + else: + self._computed_FIM = _compute_fim_for_model(model) + self._computed_FIM_by_experiment = [np.array(self._computed_FIM, copy=True)] return self._computed_FIM + def _analytic_FIM(self, model=None): + """Compute the FIM using analytic or automatic sensitivities. + + This helper is used for gradient methods that do not require + finite-difference scenarios. For ``pynumero``, sensitivities are + constructed through :class:`ExperimentGradients`. For ``kaug``, + sensitivities are extracted from the K_AUG interface. + """ + if model is None: + self.compute_FIM_model = self.experiment.get_labeled_model( + **self.get_labeled_model_args + ).clone() + model = self.compute_FIM_model + + if not hasattr(model, "objective"): + model.objective = pyo.Objective(expr=0, sense=pyo.minimize) + + for comp in model.experiment_inputs: + comp.fix() + + self.solver.solve(model, tee=self.tee) + + if self._gradient_method == GradientMethod.pynumero: + experiment_grad = ExperimentGradients(model, automatic=True, symbolic=True) + jac = experiment_grad.compute_gradient_outputs_wrt_unknown_parameters() + if self.scale_nominal_param_value: + for i, (_, v) in enumerate(model.unknown_parameters.items()): + jac[:, i] *= v + if self.scale_constant_value: + jac *= self.scale_constant_value + self.kaug_jac = jac + else: + params_dict = {k.name: v for k, v in model.unknown_parameters.items()} + params_names = list(params_dict.keys()) + dsdp_re, col = get_dsdp(model, params_names, params_dict, tee=self.tee) + dsdp_array = dsdp_re.toarray().T + + dsdp_extract = [] + for k, _ in model.experiment_outputs.items(): + name = k.name + try: + kaug_no = col.index(name) + dsdp_extract.append(dsdp_array[kaug_no]) + except Exception: + dsdp_extract.append(np.zeros(len(params_names))) + + jac = [[] for _ in params_names] + for d in range(len(dsdp_extract)): + for k, v in model.unknown_parameters.items(): + p = params_names.index(k.name) + sensi = dsdp_extract[d][p] * self.scale_constant_value + if self.scale_nominal_param_value: + sensi *= v + jac[p].append(sensi) + self.kaug_jac = np.array(jac).T + + cov_y = np.zeros((len(model.measurement_error), len(model.measurement_error))) + count = 0 + for _, v in model.measurement_error.items(): + cov_y[count, count] = 1 / v**2 + count += 1 + + self.analytic_FIM = self.kaug_jac.T @ cov_y @ self.kaug_jac + self.prior_FIM + + def _validated_fd_formula(self): + """Return the validated finite-difference scheme for FD gradients.""" + if not isinstance(self.fd_formula, FiniteDifferenceStep): + raise DeveloperError( + "Finite difference option not recognized. Please contact " + "the developers as you should not see this error." + ) + return self.fd_formula + # Use a sequential method to get the FIM def _sequential_FIM(self, model=None): """ @@ -622,11 +2534,20 @@ def _sequential_FIM(self, model=None): """ # Build a single model instance if model is None: - self.compute_FIM_model = self.experiment.get_labeled_model( - **self.get_labeled_model_args - ).clone() + self.compute_FIM_model = ( + self.experiment_list[0] + .get_labeled_model(**self.get_labeled_model_args) + .clone() + ) model = self.compute_FIM_model + fd_formula = ( + self._validated_fd_formula() + if self._gradient_method + in [GradientMethod.forward, GradientMethod.central, GradientMethod.backward] + else None + ) + # Create suffix to keep track of parameter scenarios if hasattr(model, "parameter_scenarios"): model.del_component(model.parameter_scenarios) @@ -634,7 +2555,7 @@ def _sequential_FIM(self, model=None): # Populate parameter scenarios, and scenario # inds based on finite difference scheme - if self.fd_formula == FiniteDifferenceStep.central: + if self._gradient_method == GradientMethod.central: model.parameter_scenarios.update( (2 * ind, k) for ind, k in enumerate(model.unknown_parameters.keys()) ) @@ -643,10 +2564,7 @@ def _sequential_FIM(self, model=None): for ind, k in enumerate(model.unknown_parameters.keys()) ) model.scenarios = range(len(model.unknown_parameters) * 2) - elif self.fd_formula in [ - FiniteDifferenceStep.forward, - FiniteDifferenceStep.backward, - ]: + elif self._gradient_method in [GradientMethod.forward, GradientMethod.backward]: model.parameter_scenarios.update( (ind + 1, k) for ind, k in enumerate(model.unknown_parameters.keys()) ) @@ -666,21 +2584,21 @@ def _sequential_FIM(self, model=None): # Calculate measurement values for each scenario for s in model.scenarios: # Perturbation to be (1 + diff) * param_value - if self.fd_formula == FiniteDifferenceStep.central: + if fd_formula == FiniteDifferenceStep.central: diff = self.step * ( (-1) ** s ) # Positive perturbation, even; negative, odd - elif self.fd_formula == FiniteDifferenceStep.backward: + elif fd_formula == FiniteDifferenceStep.backward: diff = ( self.step * -1 * (s != 0) ) # Backward always negative perturbation; 0 at s = 0 - elif self.fd_formula == FiniteDifferenceStep.forward: + elif fd_formula == FiniteDifferenceStep.forward: diff = self.step * (s != 0) # Forward always positive; 0 at s = 0 # If we are doing forward/backward, no change for s=0 skip_param_update = ( - self.fd_formula - in [FiniteDifferenceStep.forward, FiniteDifferenceStep.backward] + self._gradient_method + in [GradientMethod.forward, GradientMethod.backward] ) and (s == 0) if not skip_param_update: param = model.parameter_scenarios[s] @@ -731,14 +2649,14 @@ def _sequential_FIM(self, model=None): for k, v in model.unknown_parameters.items(): curr_step = v * self.step - if self.fd_formula == FiniteDifferenceStep.central: + if fd_formula == FiniteDifferenceStep.central: col_1 = 2 * i col_2 = 2 * i + 1 curr_step *= 2 - elif self.fd_formula == FiniteDifferenceStep.forward: + elif fd_formula == FiniteDifferenceStep.forward: col_1 = i col_2 = 0 - elif self.fd_formula == FiniteDifferenceStep.backward: + elif fd_formula == FiniteDifferenceStep.backward: col_1 = 0 col_2 = i @@ -783,9 +2701,11 @@ def _kaug_FIM(self, model=None): # Remake compute_FIM_model if model is None. # compute_FIM_model needs to be the right version for function to work. if model is None: - self.compute_FIM_model = self.experiment.get_labeled_model( - **self.get_labeled_model_args - ).clone() + self.compute_FIM_model = ( + self.experiment_list[0] + .get_labeled_model(**self.get_labeled_model_args) + .clone() + ) model = self.compute_FIM_model # add zero (dummy/placeholder) objective function @@ -864,7 +2784,9 @@ def _kaug_FIM(self, model=None): self.kaug_FIM = self.kaug_jac.T @ cov_y @ self.kaug_jac + self.prior_FIM # Create the DoE model (with ``scenarios`` from finite differencing scheme) - def create_doe_model(self, model=None): + def create_doe_model( + self, model=None, experiment_index=0, _for_multi_experiment=False + ): """ Add equations to compute sensitivities, FIM, and objective. Builds the DoE model. Adds the scenarios, the sensitivity matrix @@ -878,6 +2800,9 @@ def create_doe_model(self, model=None): Parameters ---------- model: model to add finite difference scenarios + experiment_index: index of experiment in experiment_list to use for this model (default: 0) + _for_multi_experiment: if True, skip creating L matrix and other objective-related + variables that will be created at the aggregated level (default: False) """ if model is None: @@ -905,28 +2830,36 @@ def create_doe_model(self, model=None): ) # Generate scenarios for finite difference formulae - self._generate_scenario_blocks(model=model) + self._generate_fd_scenario_blocks( + model=model, experiment_index=experiment_index + ) # Set names for indexing sensitivity matrix (jacobian) and FIM scen_block_ind = min( [ - k.name.split(".").index("scenario_blocks[0]") - for k in model.scenario_blocks[0].unknown_parameters.keys() + k.name.split(".").index("fd_scenario_blocks[0]") + for k in model.fd_scenario_blocks[0].unknown_parameters.keys() ] ) model.parameter_names = pyo.Set( initialize=[ ".".join(k.name.split(".")[(scen_block_ind + 1) :]) - for k in model.scenario_blocks[0].unknown_parameters.keys() + for k in model.fd_scenario_blocks[0].unknown_parameters.keys() ] ) model.output_names = pyo.Set( initialize=[ ".".join(k.name.split(".")[(scen_block_ind + 1) :]) - for k in model.scenario_blocks[0].experiment_outputs.keys() + for k in model.fd_scenario_blocks[0].experiment_outputs.keys() ] ) + experiment_grad = None + if self._gradient_method == GradientMethod.pynumero: + experiment_grad = ExperimentGradients( + model.fd_scenario_blocks[0], automatic=True, symbolic=True + ) + def identity_matrix(m, i, j): if i == j: return 1 @@ -1005,9 +2938,10 @@ def initialize_fim_inv(m, j, d): # To-Do: Look into this functionality..... # if cholesky, define L elements as variables - if self.Cholesky_option and self.objective_option in ( - ObjectiveLib.determinant, - ObjectiveLib.trace, + if ( + not _for_multi_experiment + and self.Cholesky_option + and self.objective_option in (ObjectiveLib.determinant, ObjectiveLib.trace) ): model.L = pyo.Var( model.parameter_names, model.parameter_names, initialize=identity_matrix @@ -1036,50 +2970,85 @@ def initialize_fim_inv(m, j, d): if self.objective_option == ObjectiveLib.trace: model.L_inv[c, d].setlb(self.L_diagonal_lower_bound) - # jacobian rule + if self._gradient_method == GradientMethod.pynumero: + experiment_grad.construct_sensitivity_constraints(model.fd_scenario_blocks[0]) + def jacobian_rule(m, n, p): """ m: Pyomo model n: experimental output p: unknown parameter """ + if self._gradient_method == GradientMethod.pynumero: + # The symbolic/pynumero path already constructs sensitivity + # variables on the base scenario block for each + # output/parameter pair. + output_cuid = pyo.ComponentUID(n) + output_var = output_cuid.find_component_on(m.fd_scenario_blocks[0]) + + parameter_cuid = pyo.ComponentUID(p) + parameter_var = parameter_cuid.find_component_on(m.fd_scenario_blocks[0]) + + i = experiment_grad.measurement_mapping[output_var] + j = experiment_grad.parameter_mapping[parameter_var] + + if i is None: + sensitivity_value = 0 + else: + sensitivity_value = m.fd_scenario_blocks[0].jac_variables_wrt_param[ + i, j + ] + + # When nominal-parameter scaling is requested, convert the + # direct sensitivity into the scaled convention used in the + # DoE model. + scale = parameter_var if self.scale_nominal_param_value else 1 + return m.sensitivity_jacobian[n, p] == ( + sensitivity_value * self.scale_constant_value * scale + ) + + # The finite-difference path reconstructs sensitivities from the + # perturbed scenario blocks associated with each parameter. fd_step_mult = 1 cuid = pyo.ComponentUID(n) param_ind = m.parameter_names.data().index(p) - # Different FD schemes lead to different scenarios for the computation - if self.fd_formula == FiniteDifferenceStep.central: + if self._gradient_method == GradientMethod.central: s1 = param_ind * 2 s2 = param_ind * 2 + 1 fd_step_mult = 2 - elif self.fd_formula == FiniteDifferenceStep.forward: + elif self._gradient_method == GradientMethod.forward: s1 = param_ind + 1 s2 = 0 - elif self.fd_formula == FiniteDifferenceStep.backward: + elif self._gradient_method == GradientMethod.backward: s1 = 0 s2 = param_ind + 1 + else: + raise DeveloperError( + "Gradient method option not recognized while building the Jacobian." + ) - var_up = cuid.find_component_on(m.scenario_blocks[s1]) - var_lo = cuid.find_component_on(m.scenario_blocks[s2]) + var_up = cuid.find_component_on(m.fd_scenario_blocks[s1]) + var_lo = cuid.find_component_on(m.fd_scenario_blocks[s2]) param = m.parameter_scenarios[max(s1, s2)] - param_loc = pyo.ComponentUID(param).find_component_on(m.scenario_blocks[0]) - param_val = m.scenario_blocks[0].unknown_parameters[param_loc] + param_loc = pyo.ComponentUID(param).find_component_on( + m.fd_scenario_blocks[0] + ) + param_val = m.fd_scenario_blocks[0].unknown_parameters[param_loc] param_diff = param_val * fd_step_mult * self.step + sensitivity_value = (var_up - var_lo) / param_diff + # Apply the same nominal-parameter scaling convention used by the + # symbolic path so the sensitivity_jacobian has a consistent + # meaning regardless of derivative backend. if self.scale_nominal_param_value: - return ( - m.sensitivity_jacobian[n, p] - == (var_up - var_lo) - / param_diff - * param_val - * self.scale_constant_value - ) - else: - return ( - m.sensitivity_jacobian[n, p] - == (var_up - var_lo) / param_diff * self.scale_constant_value - ) + sensitivity_value = sensitivity_value * param_val + + return ( + m.sensitivity_jacobian[n, p] + == sensitivity_value * self.scale_constant_value + ) # A constraint to calculate elements in Hessian matrix # transfer prior FIM to be Expressions @@ -1117,20 +3086,38 @@ def fim_rule(m, p, q): else: return m.fim[p, q] == m.fim[q, p] else: - return ( - m.fim[p, q] - == sum( + # For multi-experiment optimization, prior_FIM is added to the + # aggregated total_fim, not to each individual experiment's FIM + if _for_multi_experiment: + return m.fim[p, q] == sum( 1 - / m.scenario_blocks[0].measurement_error[ - pyo.ComponentUID(n).find_component_on(m.scenario_blocks[0]) + / m.fd_scenario_blocks[0].measurement_error[ + pyo.ComponentUID(n).find_component_on( + m.fd_scenario_blocks[0] + ) ] ** 2 * m.sensitivity_jacobian[n, p] * m.sensitivity_jacobian[n, q] for n in m.output_names ) - + m.prior_FIM[p, q] - ) + else: + return ( + m.fim[p, q] + == sum( + 1 + / m.fd_scenario_blocks[0].measurement_error[ + pyo.ComponentUID(n).find_component_on( + m.fd_scenario_blocks[0] + ) + ] + ** 2 + * m.sensitivity_jacobian[n, p] + * m.sensitivity_jacobian[n, q] + for n in m.output_names + ) + + m.prior_FIM[p, q] + ) model.jacobian_constraint = pyo.Constraint( model.output_names, model.parameter_names, rule=jacobian_rule @@ -1151,7 +3138,7 @@ def fim_rule(m, p, q): model.fim_inv[p, q].fix(0.0) # Create scenario block structure - def _generate_scenario_blocks(self, model=None): + def _generate_fd_scenario_blocks(self, model=None, experiment_index=0): """ Generates the modeling blocks corresponding to the scenarios for the finite differencing scheme to compute the sensitivity jacobian @@ -1165,15 +3152,25 @@ def _generate_scenario_blocks(self, model=None): Parameters ---------- model: model to add finite difference scenarios + experiment_index: index of experiment in experiment_list to use for this model (default: 0) """ # If model is none, assume it is self.model if model is None: model = self.model + fd_formula = ( + self._validated_fd_formula() + if self._gradient_method + in [GradientMethod.forward, GradientMethod.central, GradientMethod.backward] + else None + ) + # Generate initial scenario to populate unknown parameter values - model.base_model = self.experiment.get_labeled_model( - **self.get_labeled_model_args - ).clone() + model.base_model = ( + self.experiment_list[experiment_index] + .get_labeled_model(**self.get_labeled_model_args) + .clone() + ) # Check the model that labels are correct self.check_model_labels(model=model.base_model) @@ -1214,7 +3211,9 @@ def _generate_scenario_blocks(self, model=None): # Populate parameter scenarios, and scenario # inds based on finite difference scheme - if self.fd_formula == FiniteDifferenceStep.central: + if self._gradient_method == GradientMethod.pynumero: + model.scenarios = range(1) + elif fd_formula == FiniteDifferenceStep.central: model.parameter_scenarios.update( (2 * ind, k) for ind, k in enumerate(model.base_model.unknown_parameters.keys()) @@ -1224,7 +3223,7 @@ def _generate_scenario_blocks(self, model=None): for ind, k in enumerate(model.base_model.unknown_parameters.keys()) ) model.scenarios = range(len(model.base_model.unknown_parameters) * 2) - elif self.fd_formula in [ + elif fd_formula in [ FiniteDifferenceStep.forward, FiniteDifferenceStep.backward, ]: @@ -1259,38 +3258,42 @@ def _generate_scenario_blocks(self, model=None): # Generate blocks for finite difference scenarios def build_block_scenarios(b, s): # Generate model for the finite difference scenario - m = b.model() - b.transfer_attributes_from(m.base_model.clone()) + # Get the parent block that contains base_model + parent_block = b.parent_block() + b.transfer_attributes_from(parent_block.base_model.clone()) + + if self._gradient_method == GradientMethod.pynumero: + return # Forward/Backward difference have a stationary # case (s == 0), no parameter to perturb - if self.fd_formula in [ + if fd_formula in [ FiniteDifferenceStep.forward, FiniteDifferenceStep.backward, ]: if s == 0: return - param = m.parameter_scenarios[s] + param = parent_block.parameter_scenarios[s] # Perturbation to be (1 + diff) * param_value - if self.fd_formula == FiniteDifferenceStep.central: + if fd_formula == FiniteDifferenceStep.central: diff = self.step * ( (-1) ** s ) # Positive perturbation, even; negative, odd - elif self.fd_formula == FiniteDifferenceStep.backward: + elif fd_formula == FiniteDifferenceStep.backward: diff = self.step * -1 # Backward always negative perturbation - elif self.fd_formula == FiniteDifferenceStep.forward: + elif fd_formula == FiniteDifferenceStep.forward: diff = self.step # Forward always positive else: - # TODO: add an error message for this as not being implemented yet - diff = 0 - pass + raise DeveloperError( + "Finite difference option not recognized while generating scenario blocks." + ) # Update parameter values for the given finite difference scenario - pyo.ComponentUID(param, context=m.base_model).find_component_on( + pyo.ComponentUID(param, context=parent_block.base_model).find_component_on( b - ).set_value(m.base_model.unknown_parameters[param] * (1 + diff)) + ).set_value(parent_block.base_model.unknown_parameters[param] * (1 + diff)) # Fix experiment inputs before solve (enforce square solve) for comp in b.experiment_inputs: @@ -1302,23 +3305,28 @@ def build_block_scenarios(b, s): for comp in b.experiment_inputs: comp.unfix() - model.scenario_blocks = pyo.Block(model.scenarios, rule=build_block_scenarios) + model.fd_scenario_blocks = pyo.Block( + model.scenarios, rule=build_block_scenarios + ) # TODO: this might have to change if experiment inputs have # a different value in the Suffix (currently it is the CUID) - design_vars = [k for k, v in model.scenario_blocks[0].experiment_inputs.items()] + design_vars = [ + k for k, v in model.fd_scenario_blocks[0].experiment_inputs.items() + ] - # Add constraints to equate block design with global design: - for ind, d in enumerate(design_vars): - con_name = "global_design_eq_con_" + str(ind) + if self._gradient_method != GradientMethod.pynumero: + # Add constraints to equate block design with global design: + for ind, d in enumerate(design_vars): + con_name = "global_design_eq_con_" + str(ind) # Constraint rule for global design constraints def global_design_fixing(m, s): if s == 0: return pyo.Constraint.Skip block_design_var = pyo.ComponentUID( - d, context=m.scenario_blocks[0] - ).find_component_on(m.scenario_blocks[s]) + d, context=m.fd_scenario_blocks[0] + ).find_component_on(m.fd_scenario_blocks[s]) return d == block_design_var model.add_component( @@ -1349,6 +3357,15 @@ def create_objective_function(self, model=None): if model is None: model = self.model + if self.objective_option in [ + ObjectiveLib.minimum_eigenvalue, + ObjectiveLib.condition_number, + ]: + raise ValueError( + f"objective_option='{self.objective_option.value}' requires " + "use_grey_box_objective=True." + ) + if self.objective_option not in [ ObjectiveLib.determinant, ObjectiveLib.trace, @@ -1372,10 +3389,11 @@ def create_objective_function(self, model=None): model.obj_cons = pyo.Block() # Assemble the FIM matrix. This is helpful for initialization! + # collect current FIM values in row-major order fim_vals = [ model.fim[bu, un].value - for i, bu in enumerate(model.parameter_names) - for j, un in enumerate(model.parameter_names) + for bu in model.parameter_names + for un in model.parameter_names ] fim = np.array(fim_vals).reshape( len(model.parameter_names), len(model.parameter_names) @@ -1411,23 +3429,10 @@ def create_objective_function(self, model=None): for j, d in enumerate(model.parameter_names): model.L_inv[c, d].value = L_inv[i, j] - def cholesky_imp(b, c, d): - """ - Calculate Cholesky L matrix using algebraic constraints - """ - # If the row is greater than or equal to the column, we are in the - # lower triangle region of the L and FIM matrices. - # This region is where our equations are well-defined. - m = b.model() - if list(m.parameter_names).index(c) >= list(m.parameter_names).index(d): - return m.fim[c, d] == sum( - m.L[c, m.parameter_names.at(k + 1)] - * m.L[d, m.parameter_names.at(k + 1)] - for k in range(list(m.parameter_names).index(d) + 1) - ) - else: - # This is the empty half of L above the diagonal - return pyo.Constraint.Skip + if self.objective_option == ObjectiveLib.trace and not self.Cholesky_option: + raise ValueError( + "objective_option='trace' currently only implemented with ``_Cholesky option=True``." + ) # If trace objective, need L inverse constraints if self.Cholesky_option and self.objective_option == ObjectiveLib.trace: @@ -1522,6 +3527,7 @@ def determinant_general(b): list_p = list(object_p) # generate a name_order to iterate \sigma_i + # NOTE: Not used in calculation. Delete? det_perm = 0 for i in range(len(list_p)): name_order = [] @@ -1546,7 +3552,11 @@ def determinant_general(b): if self.Cholesky_option and self.objective_option == ObjectiveLib.determinant: model.obj_cons.cholesky_cons = pyo.Constraint( - model.parameter_names, model.parameter_names, rule=cholesky_imp + model.parameter_names, + model.parameter_names, + rule=self._make_cholesky_rule( + model.fim, model.L, model.parameter_names + ), ) model.objective = pyo.Objective( expr=2 * sum(pyo.log10(model.L[j, j]) for j in model.parameter_names), @@ -1565,17 +3575,17 @@ def determinant_general(b): ) elif self.objective_option == ObjectiveLib.trace: - if not self.Cholesky_option: - raise ValueError( - "objective_option='trace' currently only implemented with ``_Cholesky option=True``." - ) # if Cholesky and trace, calculating # the OBJ with trace model.cov_trace = pyo.Var( initialize=np.trace(np.linalg.inv(fim)), bounds=(small_number, None) ) model.obj_cons.cholesky_cons = pyo.Constraint( - model.parameter_names, model.parameter_names, rule=cholesky_imp + model.parameter_names, + model.parameter_names, + rule=self._make_cholesky_rule( + model.fim, model.L, model.parameter_names + ), ) model.obj_cons.cholesky_inv_cons = pyo.Constraint( model.parameter_names, model.parameter_names, rule=cholesky_inv_imp @@ -1612,11 +3622,381 @@ def determinant_general(b): # add dummy objective function model.objective = pyo.Objective(expr=0) - def create_grey_box_objective_function(self, model=None): + def create_multi_experiment_objective_function(self, model): + """ + Create objective for multi-experiment optimization. + + For each scenario s: + 1. Creates total_fim[s] = sum of exp_blocks[k].fim + prior_FIM + 2. Creates Cholesky/determinant/trace variables and constraints per scenario + 3. Creates single top-level objective summing across scenarios + + Parameters + ---------- + model: model with param_scenario_blocks structure + """ + # Validate objective option for multi-experiment. + if self.use_grey_box: + if self.objective_option in [ObjectiveLib.pseudo_trace, ObjectiveLib.zero]: + raise ValueError( + "Grey-box objective support in optimize_experiments() is only " + "available for objective_option in ['determinant', 'trace', " + "'minimum_eigenvalue', 'condition_number']." + ) + self._grey_box_output_name() + else: + if self.objective_option in [ + ObjectiveLib.minimum_eigenvalue, + ObjectiveLib.condition_number, + ]: + raise ValueError( + f"objective_option='{self.objective_option.value}' requires " + "use_grey_box_objective=True." + ) + if self.objective_option not in [ + ObjectiveLib.determinant, + ObjectiveLib.trace, + ObjectiveLib.pseudo_trace, + ObjectiveLib.zero, + ]: + raise DeveloperError( + "Objective option not recognized for multi-experiment optimization. " + "Please contact the developers as you should not see this error." + ) + + # Validate trace objective requires Cholesky option + if ( + not self.use_grey_box + and self.objective_option == ObjectiveLib.trace + and not self.Cholesky_option + ): + raise ValueError( + "objective_option='trace' currently only implemented with " + "``_Cholesky_option=True`` or ``use_grey_box_objective=True``." + ) + + small_number = 1e-10 + n_scenarios = len(model.param_scenario_blocks) + + # Get weights from instance attribute (set in optimize_experiments) and + # default to uniform weights if not provided + # retrieve weights; default to uniform tuple of appropriate length + default_weights = tuple([1.0 / n_scenarios] * n_scenarios) + scenario_weights = getattr(self, 'scenario_weights', default_weights) + + # Get parameter names from first experiment (same across all) + parameter_names = model.param_scenario_blocks[0].exp_blocks[0].parameter_names + n_exp = len(model.param_scenario_blocks[0].exp_blocks) + + # For each scenario: create aggregated FIM and constraints + for s in range(n_scenarios): + scenario = model.param_scenario_blocks[s] + + # 1. Create aggregated FIM variable for each scenario: + # total_fim = sum of all exp FIMs + prior_FIM + scenario.total_fim = pyo.Var(parameter_names, parameter_names) + + # 2. Constraint: total_fim[p,q] = sum_k (exp_blocks[k].fim[p,q]) + # + prior_FIM[p,q] + def total_fim_rule(b, p, q): + p_idx = list(parameter_names).index(p) + q_idx = list(parameter_names).index(q) + + # When only_compute_fim_lower=True, only compute lower triangle + # Upper triangle elements will be handled through symmetry + if self.only_compute_fim_lower and p_idx < q_idx: + return pyo.Constraint.Skip + + return b.total_fim[p, q] == ( + sum(b.exp_blocks[k].fim[p, q] for k in range(n_exp)) + + self.prior_FIM[p_idx, q_idx] + ) + + scenario.total_fim_cons = pyo.Constraint( + parameter_names, parameter_names, rule=total_fim_rule + ) + + # 3. Fix upper triangle elements to 0 if only_compute_fim_lower=True + # Initialize lower triangle from sum of individual FIMs + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + if self.only_compute_fim_lower and i < j: + # Fix upper triangle to 0 + scenario.total_fim[p, q].fix(0.0) + else: + # Initialize lower triangle and diagonal + fim_sum = sum( + pyo.value(scenario.exp_blocks[k].fim[p, q]) or 0 + for k in range(n_exp) + ) + scenario.total_fim[p, q].value = fim_sum + self.prior_FIM[i, j] + + # 4. Build the objective representation for this scenario. + if self.use_grey_box: + # In multi-experiment mode, the grey box should see the + # aggregated scenario FIM rather than any one experiment block. + # That keeps the objective mathematically aligned with the + # public API while reusing the same grey box implementation as + # the single-experiment path. + self.create_grey_box_objective_function( + model=scenario, + fim_expr=scenario.total_fim, + parameter_names=parameter_names, + build_objective=False, + ) + # 5. Create variables and constraints (initialization will happen after square solve) + elif ( + self.Cholesky_option + and self.objective_option == ObjectiveLib.determinant + ): + # (similar to single-experiment case in create_objective_function) + scenario.obj_cons = pyo.Block() + # Add lower triangular Cholesky variables per scenario + scenario.obj_cons.L = pyo.Var(parameter_names, parameter_names) + + # Fix upper triangle to 0 and set lower bound on diagonal + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + # Fix upper triangle to 0 + if i < j: + scenario.obj_cons.L[p, q].fix(0.0) + # Lower bound on diagonal + elif i == j and self.L_diagonal_lower_bound: + scenario.obj_cons.L[p, q].setlb(self.L_diagonal_lower_bound) + + # reuse shared helper to create the constraint rule + cholesky_rule = self._make_cholesky_rule( + scenario.total_fim, scenario.obj_cons.L, parameter_names + ) + scenario.obj_cons.cholesky_cons = pyo.Constraint( + parameter_names, parameter_names, rule=cholesky_rule + ) + + elif self.Cholesky_option and self.objective_option == ObjectiveLib.trace: + scenario.obj_cons = pyo.Block() + # Add lower triangular Cholesky variables per scenario + scenario.obj_cons.L = pyo.Var(parameter_names, parameter_names) + scenario.obj_cons.L_inv = pyo.Var(parameter_names, parameter_names) + scenario.obj_cons.fim_inv = pyo.Var(parameter_names, parameter_names) + scenario.obj_cons.cov_trace = pyo.Var(bounds=(small_number, None)) + + # Fix upper triangle of L and L_inv to 0 and set lower bound on diagonal + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + # Fix upper triangle to 0 + if i < j: + scenario.obj_cons.L[p, q].fix(0.0) + scenario.obj_cons.L_inv[p, q].fix(0.0) + # Lower bound on diagonal + elif i == j and self.L_diagonal_lower_bound: + scenario.obj_cons.L[p, q].setlb(self.L_diagonal_lower_bound) + + # reuse shared helper to create the constraint rule + cholesky_rule = self._make_cholesky_rule( + scenario.total_fim, scenario.obj_cons.L, parameter_names + ) + scenario.obj_cons.cholesky_cons = pyo.Constraint( + parameter_names, parameter_names, rule=cholesky_rule + ) + + # reuse helpers for the inverse and identity rules instead of + # re-implementing the logic in-place + cholesky_inv_rule = self._make_cholesky_inv_rule( + scenario.obj_cons.fim_inv, scenario.obj_cons.L_inv, parameter_names + ) + + cholesky_LLinv_rule = self._make_cholesky_LLinv_rule( + scenario.obj_cons.L, scenario.obj_cons.L_inv, parameter_names + ) + + # Covariance trace calculation + def cov_trace_rule(b): + return b.cov_trace == sum(b.fim_inv[j, j] for j in parameter_names) + + # Add all constraints + scenario.obj_cons.cholesky_inv_cons = pyo.Constraint( + parameter_names, parameter_names, rule=cholesky_inv_rule + ) + scenario.obj_cons.cholesky_LLinv_cons = pyo.Constraint( + parameter_names, parameter_names, rule=cholesky_LLinv_rule + ) + scenario.obj_cons.cov_trace_cons = pyo.Constraint(rule=cov_trace_rule) + + # Optional: improve Cholesky roundoff error + if self.improve_cholesky_roundoff_error: + + def cholesky_fim_diag(b, p, q): + return scenario.total_fim[p, p] >= b.L[p, q] ** 2 + + def cholesky_fim_inv_diag(b, p, q): + return b.fim_inv[p, p] >= b.L_inv[p, q] ** 2 + + scenario.obj_cons.cholesky_fim_diag_cons = pyo.Constraint( + parameter_names, parameter_names, rule=cholesky_fim_diag + ) + scenario.obj_cons.cholesky_fim_inv_diag_cons = pyo.Constraint( + parameter_names, parameter_names, rule=cholesky_fim_inv_diag + ) + + elif self.objective_option == ObjectiveLib.determinant: + scenario.obj_cons = pyo.Block() + # Non-Cholesky determinant: create determinant var per scenario + scenario.obj_cons.determinant = pyo.Var(bounds=(small_number, None)) + + # Determinant constraint (explicit formula) + def determinant_general(b): + r_list = list(range(len(parameter_names))) + object_p = permutations(r_list) + list_p = list(object_p) + + det_perm = sum( + self._sgn(list_p[d]) + * math.prod( + scenario.total_fim[ + parameter_names.at(val + 1), parameter_names.at(ind + 1) + ] + for ind, val in enumerate(list_p[d]) + ) + for d in range(len(list_p)) + ) + return b.determinant == det_perm + + scenario.obj_cons.determinant_cons = pyo.Constraint( + rule=determinant_general + ) + + elif self.objective_option == ObjectiveLib.pseudo_trace: + scenario.obj_cons = pyo.Block() + # Pseudo trace objective (Trace of FIM) + scenario.obj_cons.pseudo_trace = pyo.Var(bounds=(small_number, None)) + + # Pseudo trace constraint + def pseudo_trace_rule(b): + return b.pseudo_trace == sum( + scenario.total_fim[j, j] for j in parameter_names + ) + + scenario.obj_cons.pseudo_trace_cons = pyo.Constraint( + rule=pseudo_trace_rule + ) + + # 5. Create single top-level objective summing across scenarios + if self.use_grey_box: + model.objective = pyo.Objective( + expr=sum( + scenario_weights[s] + * model.param_scenario_blocks[s].obj_cons.egb_fim_block.outputs[ + self._grey_box_output_name() + ] + for s in range(n_scenarios) + ), + sense=( + pyo.maximize + if self.objective_option in self._MAXIMIZE_OBJECTIVES + else pyo.minimize + ), + ) + + elif self.Cholesky_option and self.objective_option == ObjectiveLib.determinant: + model.objective = pyo.Objective( + expr=sum( + scenario_weights[s] + * 2 + * sum( + pyo.log10(model.param_scenario_blocks[s].obj_cons.L[j, j]) + for j in parameter_names + ) + for s in range(n_scenarios) + ), + sense=pyo.maximize, + ) + + elif self.objective_option == ObjectiveLib.determinant: + model.objective = pyo.Objective( + expr=sum( + scenario_weights[s] + * pyo.log10( + model.param_scenario_blocks[s].obj_cons.determinant + + _SMALL_TOLERANCE_DEFINITENESS # to avoid log(0) + ) + for s in range(n_scenarios) + ), + sense=pyo.maximize, + ) + + elif self.Cholesky_option and self.objective_option == ObjectiveLib.trace: + model.objective = pyo.Objective( + expr=sum( + scenario_weights[s] + * model.param_scenario_blocks[s].obj_cons.cov_trace + for s in range(n_scenarios) + ), + sense=pyo.minimize, + ) + + elif self.objective_option == ObjectiveLib.pseudo_trace: + model.objective = pyo.Objective( + expr=sum( + scenario_weights[s] + * pyo.log10( + model.param_scenario_blocks[s].obj_cons.pseudo_trace + + _SMALL_TOLERANCE_DEFINITENESS # to avoid log(0) + ) + for s in range(n_scenarios) + ), + sense=pyo.maximize, + ) + + elif self.objective_option == ObjectiveLib.zero: + # Dummy objective + model.objective = pyo.Objective(expr=0) + + def create_grey_box_objective_function( + self, model=None, fim_expr=None, parameter_names=None, build_objective=True + ): + """ + Attach the FIM grey box block to a model or scenario block. + + The same helper is used for both public APIs. In the single-experiment + path it connects to ``model.fim`` and also creates the top-level + objective. In multi-experiment mode it connects to ``scenario.total_fim`` + so each scenario contributes its aggregated FIM metric to the outer + objective sum. + """ # Add external grey box block to a block named ``obj_cons`` to # reuse material for initializing the objective-free square model if model is None: - model = model = self.model + model = self.model + if fim_expr is None: + if not hasattr(model, "fim"): + raise RuntimeError( + "Model provided does not have variable `fim`. Please make " + "sure the model is built properly before creating the grey " + "box objective." + ) + fim_expr = model.fim + if parameter_names is None: + if not hasattr(model, "parameter_names"): + raise RuntimeError( + "Model provided does not define `parameter_names`. Please " + "make sure the model is built properly before creating the " + "grey box objective." + ) + parameter_names = model.parameter_names + + output_name = self._grey_box_output_name() + + # The external model expects a dense symmetric FIM as its initializer. + # When the algebraic model stores only the lower triangle, mirror those + # values here so the grey box starts from the same physical matrix. + fim_vals = [ + pyo.value(fim_expr[p, q]) for p in parameter_names for q in parameter_names + ] + fim_initial = np.array(fim_vals, dtype=np.float64).reshape( + (len(parameter_names), len(parameter_names)) + ) + if self.only_compute_fim_lower: + fim_initial = self._symmetrize_lower_tri(fim_initial) # TODO: Make this naming convention robust model.obj_cons = pyo.Block() @@ -1624,6 +4004,8 @@ def create_grey_box_objective_function(self, model=None): # Create FIM External Grey Box object grey_box_FIM = FIMExternalGreyBox( doe_object=self, + parameter_names=parameter_names, + fim_initial=fim_initial, objective_option=self.objective_option, logger_level=self.logger.getEffectiveLevel(), ) @@ -1642,43 +4024,29 @@ def FIM_egb_cons(m, p1, p2): p2: parameter 2 """ - # Using upper triangular FIM to - # make numerics better. - if list(model.parameter_names).index(p1) >= list( - model.parameter_names - ).index(p2): - return model.fim[(p1, p2)] == m.egb_fim_block.inputs[(p2, p1)] + # The grey box receives only the triangular portion of the FIM. We + # therefore map the lower-triangular algebraic entries into the + # upper-triangular input names expected by ExternalGreyBoxBlock. + if list(parameter_names).index(p1) >= list(parameter_names).index(p2): + return fim_expr[(p1, p2)] == m.egb_fim_block.inputs[(p2, p1)] else: return pyo.Constraint.Skip # Add the FIM and External Grey # Box inputs constraints model.obj_cons.FIM_equalities = pyo.Constraint( - model.parameter_names, model.parameter_names, rule=FIM_egb_cons + parameter_names, parameter_names, rule=FIM_egb_cons ) - # Add objective based on user provided - # type within ObjectiveLib - if self.objective_option == ObjectiveLib.trace: - model.objective = pyo.Objective( - expr=model.obj_cons.egb_fim_block.outputs["A-opt"], sense=pyo.minimize - ) - elif self.objective_option == ObjectiveLib.determinant: - model.objective = pyo.Objective( - expr=model.obj_cons.egb_fim_block.outputs["log-D-opt"], - sense=pyo.maximize, - ) - elif self.objective_option == ObjectiveLib.minimum_eigenvalue: - model.objective = pyo.Objective( - expr=model.obj_cons.egb_fim_block.outputs["E-opt"], sense=pyo.maximize - ) - elif self.objective_option == ObjectiveLib.condition_number: + if build_objective: model.objective = pyo.Objective( - expr=model.obj_cons.egb_fim_block.outputs["ME-opt"], sense=pyo.minimize + expr=model.obj_cons.egb_fim_block.outputs[output_name], + sense=( + pyo.maximize + if self.objective_option in self._MAXIMIZE_OBJECTIVES + else pyo.minimize + ), ) - # Else error not needed for spurious objective - # options as the error will always appear - # when creating the FIMExternalGreyBox block # Check to see if the model has all the required suffixes def check_model_labels(self, model=None): @@ -1699,30 +4067,71 @@ def check_model_labels(self, model=None): "Experiment model does not have suffix " + '"experiment_outputs".' ) + # Check that experiment_outputs is not empty + if len(outputs) == 0: + raise ValueError( + "No experiment outputs found. Design of Experiments requires at least " + "one experiment output (measurement) to optimize. Please add an " + "'experiment_outputs' Suffix to your model with at least one variable." + ) + # Check that experimental inputs exist try: - outputs = [k.name for k, v in model.experiment_inputs.items()] + inputs = [k.name for k, v in model.experiment_inputs.items()] except: raise RuntimeError( "Experiment model does not have suffix " + '"experiment_inputs".' ) + # Check that experiment_inputs is not empty + if len(inputs) == 0: + raise ValueError( + "No experiment inputs found. Design of Experiments requires at least " + "one experiment input (design variable) to optimize. Please add an " + "'experiment_inputs' Suffix to your model with at least one variable." + ) + # Check that unknown parameters exist try: - outputs = [k.name for k, v in model.unknown_parameters.items()] + parameters = [k.name for k, v in model.unknown_parameters.items()] except: raise RuntimeError( "Experiment model does not have suffix " + '"unknown_parameters".' ) + # Check that unknown_parameters is not empty + if len(parameters) == 0: + raise ValueError( + "No unknown parameters found. Design of Experiments requires at least " + "one unknown parameter to estimate. Please add an " + "'unknown_parameters' Suffix to your model with at least one variable." + ) + # Check that measurement errors exist try: - outputs = [k.name for k, v in model.measurement_error.items()] + errors = [k.name for k, v in model.measurement_error.items()] except: raise RuntimeError( "Experiment model does not have suffix " + '"measurement_error".' ) + # Check that measurement_error is not empty + if len(errors) == 0: + raise ValueError( + "No measurement errors found. Design of Experiments requires at least " + "one measurement error specification. Please add a " + "'measurement_error' Suffix to your model with at least one variable." + ) + + # Check that measurement_error and experiment_outputs have the same length + if len(errors) != len(outputs): + raise ValueError( + "Number of experiment outputs, {}, and length of measurement error, " + "{}, do not match. Please check model labeling.".format( + len(outputs), len(errors) + ) + ) + self.logger.info("Model has expected labels.") # Check the FIM shape against what is expected from the model. @@ -1868,9 +4277,11 @@ def compute_FIM_full_factorial( self.logger.info("Beginning Full Factorial Design.") # Make new model for factorial design - self.factorial_model = self.experiment.get_labeled_model( - **self.get_labeled_model_args - ).clone() + self.factorial_model = ( + self.experiment_list[0] + .get_labeled_model(**self.get_labeled_model_args) + .clone() + ) model = self.factorial_model # Permute the inputs to be aligned with the experiment input indices @@ -2643,7 +5054,7 @@ def get_experiment_input_values(self, model=None): model = self.model if not hasattr(model, "experiment_inputs"): - if not hasattr(model, "scenario_blocks"): + if not hasattr(model, "fd_scenario_blocks"): raise RuntimeError( "Model provided does not have expected structure. " "Please make sure model is built properly before " @@ -2652,7 +5063,7 @@ def get_experiment_input_values(self, model=None): d_vals = [ pyo.value(k) - for k, v in model.scenario_blocks[0].experiment_inputs.items() + for k, v in model.fd_scenario_blocks[0].experiment_inputs.items() ] else: d_vals = [pyo.value(k) for k, v in model.experiment_inputs.items()] @@ -2680,7 +5091,7 @@ def get_unknown_parameter_values(self, model=None): model = self.model if not hasattr(model, "unknown_parameters"): - if not hasattr(model, "scenario_blocks"): + if not hasattr(model, "fd_scenario_blocks"): raise RuntimeError( "Model provided does not have expected structure. Please make " "sure model is built properly before calling " @@ -2689,7 +5100,7 @@ def get_unknown_parameter_values(self, model=None): theta_vals = [ pyo.value(k) - for k, v in model.scenario_blocks[0].unknown_parameters.items() + for k, v in model.fd_scenario_blocks[0].unknown_parameters.items() ] else: theta_vals = [pyo.value(k) for k, v in model.unknown_parameters.items()] @@ -2716,7 +5127,7 @@ def get_experiment_output_values(self, model=None): model = self.model if not hasattr(model, "experiment_outputs"): - if not hasattr(model, "scenario_blocks"): + if not hasattr(model, "fd_scenario_blocks"): raise RuntimeError( "Model provided does not have expected structure. Please make " "sure model is built properly before calling " @@ -2725,7 +5136,7 @@ def get_experiment_output_values(self, model=None): y_hat_vals = [ pyo.value(k) - for k, v in model.scenario_blocks[0].experiment_outputs.items() + for k, v in model.fd_scenario_blocks[0].experiment_outputs.items() ] else: y_hat_vals = [pyo.value(k) for k, v in model.experiment_outputs.items()] @@ -2754,7 +5165,7 @@ def get_measurement_error_values(self, model=None): model = self.model if not hasattr(model, "measurement_error"): - if not hasattr(model, "scenario_blocks"): + if not hasattr(model, "fd_scenario_blocks"): raise RuntimeError( "Model provided does not have expected structure. Please make " "sure model is built properly before calling " @@ -2763,7 +5174,7 @@ def get_measurement_error_values(self, model=None): sigma_vals = [ pyo.value(k) - for k, v in model.scenario_blocks[0].measurement_error.items() + for k, v in model.fd_scenario_blocks[0].measurement_error.items() ] else: sigma_vals = [pyo.value(k) for k, v in model.measurement_error.items()] diff --git a/pyomo/contrib/doe/examples/polynomial.py b/pyomo/contrib/doe/examples/polynomial.py new file mode 100644 index 00000000000..dfbb51d6361 --- /dev/null +++ b/pyomo/contrib/doe/examples/polynomial.py @@ -0,0 +1,95 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + +import pyomo.environ as pyo + +from pyomo.contrib.doe import DesignOfExperiments +from pyomo.contrib.parmest.experiment import Experiment + + +class PolynomialExperiment(Experiment): + """A small algebraic experiment used for symbolic-gradient testing.""" + + def __init__(self): + self.model = None + + def get_labeled_model(self): + """Build and label the experiment model on first access.""" + if self.model is None: + self.create_model() + self.label_experiment() + return self.model + + def create_model(self): + """Define a polynomial model for testing symbolic sensitivities. + + y = a*x1 + b*x2 + c*x1*x2 + d + """ + + m = self.model = pyo.ConcreteModel() + + m.x1 = pyo.Var(bounds=(-5, 5), initialize=2.0) + m.x2 = pyo.Var(bounds=(-5, 5), initialize=3.0) + + m.a = pyo.Var(bounds=(-5, 5), initialize=2) + m.b = pyo.Var(bounds=(-5, 5), initialize=-1) + m.c = pyo.Var(bounds=(-5, 5), initialize=0.5) + m.d = pyo.Var(bounds=(-5, 5), initialize=-1) + + m.y = pyo.Var(initialize=0) + + @m.Constraint() + def output_equation(m): + return m.y == m.a * m.x1 + m.b * m.x2 + m.c * m.x1 * m.x2 + m.d + + def label_experiment(self): + """Attach the standard DoE suffixes to the polynomial model.""" + m = self.model + + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs[m.y] = None + + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error[m.y] = 1 + + m.experiment_inputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_inputs[m.x1] = None + m.experiment_inputs[m.x2] = None + + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update((k, pyo.value(k)) for k in [m.a, m.b, m.c, m.d]) + + +def run_polynomial_doe(): + """Run a small symbolic DoE FIM calculation for the polynomial model.""" + experiment = PolynomialExperiment() + + doe_obj = DesignOfExperiments( + experiment, + gradient_method="pynumero", + step=1e-3, + objective_option="determinant", + scale_constant_value=1, + scale_nominal_param_value=False, + prior_FIM=None, + jac_initial=None, + fim_initial=None, + L_diagonal_lower_bound=1e-7, + solver=pyo.SolverFactory("ipopt"), + tee=False, + get_labeled_model_args=None, + _Cholesky_option=True, + _only_compute_fim_lower=True, + ) + + return doe_obj.compute_FIM() + + +if __name__ == "__main__": + run_polynomial_doe() diff --git a/pyomo/contrib/doe/grey_box_utilities.py b/pyomo/contrib/doe/grey_box_utilities.py index 41e5cf2d78c..c34954cd2ff 100644 --- a/pyomo/contrib/doe/grey_box_utilities.py +++ b/pyomo/contrib/doe/grey_box_utilities.py @@ -44,7 +44,14 @@ class FIMExternalGreyBox( ExternalGreyBoxModel if (scipy_available and numpy_available) else object ): - def __init__(self, doe_object, objective_option="determinant", logger_level=None): + def __init__( + self, + doe_object=None, + objective_option="determinant", + logger_level=None, + parameter_names=None, + fim_initial=None, + ): """ Grey box model for metrics on the FIM. This methodology reduces numerical complexity for the computation of FIM metrics related @@ -68,17 +75,35 @@ def __init__(self, doe_object, objective_option="determinant", logger_level=None default: None, or equivalently, use the logging level of doe_object. NOTE: Use logging.DEBUG for all messages. + parameter_names: + Optional ordered iterable of parameter labels. When provided, this + lets the grey box object operate on any FIM source with the same + ordering instead of assuming the data must come from + ``doe_object.model.parameter_names``. This is needed for the + multi-experiment grey box path because the linked FIM lives on a + scenario block (``scenario.total_fim``), while ``doe_object.model`` + is the top-level container and does not own ``parameter_names`` + directly. + fim_initial: + Optional dense, symmetric FIM used to seed the grey box inputs. This + is required when ``doe_object`` is not provided. """ - if doe_object is None: + if doe_object is None and (parameter_names is None or fim_initial is None): raise ValueError( - "DoE Object must be provided to build external grey box of the FIM." + "Either ``doe_object`` or both ``parameter_names`` and " + "``fim_initial`` must be provided to build the FIM grey box." ) self.doe_object = doe_object - # Grab parameter list from the doe_object model - self._param_names = [i for i in self.doe_object.model.parameter_names] + # Grab parameter ordering from the explicit arguments when available. + # Multi-experiment optimization passes the aggregated scenario FIM + # directly, so we should not assume the linked FIM always shares the + # same location as self.doe_object.model. + if parameter_names is None: + parameter_names = self.doe_object.model.parameter_names + self._param_names = [i for i in parameter_names] self._n_params = len(self._param_names) # Check if the doe_object has model components that are required @@ -93,15 +118,21 @@ def __init__(self, doe_object, objective_option="determinant", logger_level=None # If logger level is None, use doe_object's logger level if logger_level is None: - logger_level = doe_object.logger.level + logger_level = ( + doe_object.logger.level if doe_object is not None else logging.WARNING + ) self.logger.setLevel(level=logger_level) # Set initial values for inputs # Need a mask structure - self._masking_matrix = np.triu(np.ones_like(self.doe_object.fim_initial)) + if fim_initial is None: + fim_initial = self.doe_object.fim_initial + fim_initial = np.asarray(fim_initial, dtype=np.float64) + + self._masking_matrix = np.triu(np.ones_like(fim_initial)) self._input_values = np.asarray( - self.doe_object.fim_initial[self._masking_matrix > 0], dtype=np.float64 + fim_initial[self._masking_matrix > 0], dtype=np.float64 ) self._n_inputs = len(self._input_values) diff --git a/pyomo/contrib/doe/rb_multi.py b/pyomo/contrib/doe/rb_multi.py new file mode 100644 index 00000000000..a41e87caf3c --- /dev/null +++ b/pyomo/contrib/doe/rb_multi.py @@ -0,0 +1,206 @@ +#### NO NEED TO CHECK THIS SCRIPT. THIS IS FOR ME TO UNDERSTAND THE RESULT AND WILL BE DELETED LATER. +"""Utility for scanning Rooney-Biegler multi-experiment FIM metrics.""" + +from pyomo.common.dependencies import ( + numpy as np, + numpy_available, + matplotlib, + matplotlib_available, +) + +from pyomo.contrib.doe import DesignOfExperiments +from pyomo.contrib.doe.tests.experiment_class_example_flags import ( + RooneyBieglerMultiExperiment, +) + + +def rb_multi(hour: np.ndarray, n_exp: int, prior_FIM): + """ + Compute Rooney-Biegler 2-experiment FIM metrics over hour pairs. + + Parameters + ---------- + hour: + Candidate experiment times. A 1-D array is expected; other shapes are + flattened in row-major order. + n_exp: + Number of experiments selected in each combination. This utility is + intentionally restricted to ``n_exp == 2`` for now. + prior_FIM: + Prior information matrix added once to every selected combination. + + Returns + ------- + dict + Dictionary containing the hour grid, pairwise total FIMs, the four + requested log10 metric matrices, and the matplotlib figure. + """ + if not numpy_available: + raise ImportError("rb_multi requires numpy.") + + hour = np.asarray(hour, dtype=float).ravel() + if hour.size == 0: + raise ValueError("`hour` must contain at least one candidate point.") + if n_exp != 2: + raise ValueError( + f"`rb_multi` currently supports only `n_exp == 2`, got {n_exp!r}." + ) + if n_exp > hour.size: + raise ValueError( + f"`n_exp`={n_exp} cannot exceed the number of candidate hours " + f"({hour.size})." + ) + + # Compute one single-experiment FIM per candidate hour and reuse it across + # every combination. This avoids repeating the expensive DOE solve for the + # same hour value. + point_fims = [] + for hour_value in hour: + experiment = RooneyBieglerMultiExperiment(hour=float(hour_value)) + doe = DesignOfExperiments( + experiment=experiment, objective_option="zero", step=1e-2, prior_FIM=None + ) + point_fims.append(np.asarray(doe.compute_FIM(method="sequential"), dtype=float)) + + fim_shape = point_fims[0].shape + prior_FIM = np.asarray(prior_FIM, dtype=float) + if prior_FIM.shape != fim_shape: + raise ValueError( + f"`prior_FIM` must have shape {fim_shape}, got {prior_FIM.shape}." + ) + + n_hours = hour.size + total_fims = np.empty((n_hours, n_hours), dtype=object) + log10_det = np.empty((n_hours, n_hours), dtype=float) + log10_trace_inv = np.empty((n_hours, n_hours), dtype=float) + log10_min_eig = np.empty((n_hours, n_hours), dtype=float) + log10_cond = np.empty((n_hours, n_hours), dtype=float) + + # Build the full hour-by-hour grid so the diagonal corresponds to running + # both experiments at the same candidate hour. + for i in range(n_hours): + for j in range(n_hours): + # if j < i: + # continue # Skip lower triangle; FIMs and metrics are symmetric in (i, j) + total_fim = prior_FIM.copy() + point_fims[i] + point_fims[j] + total_fims[i, j] = total_fim + + sign, logdet = np.linalg.slogdet(total_fim) + log10_det[i, j] = logdet / np.log(10.0) if sign > 0 else np.nan + + eigvals = np.linalg.eigvalsh(total_fim) + min_eig = np.min(eigvals) + log10_min_eig[i, j] = np.log10(min_eig) if min_eig > 0 else np.nan + + try: + trace_inv = np.trace(np.linalg.inv(total_fim)) + log10_trace_inv[i, j] = ( + np.log10(trace_inv) + if np.isfinite(trace_inv) and trace_inv > 0 + else np.nan + ) + except np.linalg.LinAlgError: + log10_trace_inv[i, j] = np.nan + + cond_value = np.linalg.cond(total_fim) + log10_cond[i, j] = ( + np.log10(cond_value) + if np.isfinite(cond_value) and cond_value > 0 + else np.nan + ) + + if not matplotlib_available: + raise ImportError("Plotting rb_multi results requires matplotlib.") + + figure, axes = matplotlib.pyplot.subplots(2, 2, figsize=(14, 10), sharex=True) + axes = axes.flatten() + + metric_specs = [ + ("log10(det(total_FIM))", log10_det, np.nanargmax), + ("log10(trace(inv(total_FIM)))", log10_trace_inv, np.nanargmin), + ("log10(min eig(total_FIM))", log10_min_eig, np.nanargmax), + ("log10(cond(total_FIM))", log10_cond, np.nanargmin), + ] + + for ax, (title, values, selector) in zip(axes, metric_specs): + image = ax.imshow(values, origin="lower", aspect="auto") + ax.set_title(title) + ax.set_ylabel("hour_1") + ax.set_xlabel("hour_2") + figure.colorbar(image, ax=ax) + + if np.isfinite(values).any(): + best_index = int(selector(values)) + best_row, best_col = np.unravel_index(best_index, values.shape) + best_hour_1 = float(hour[best_row]) + best_hour_2 = float(hour[best_col]) + best_value = float(values[best_row, best_col]) + + ax.plot(best_col, best_row, marker="*", color="red", markersize=14) + # Keep the annotation box inside the axes so corner optima do not + # push the label outside the subplot or into neighboring panels. + ax.text( + 0.03, + 0.97, + f"hours=({best_hour_1:.5g}, {best_hour_2:.5g})\nvalue={best_value:.5f}", + transform=ax.transAxes, + ha="left", + va="top", + color="red", + bbox={"boxstyle": "round,pad=0.2", "fc": "white", "alpha": 0.8}, + ) + + for ax in axes: + ax.set_xticks(np.arange(n_hours)) + ax.set_xticklabels([f"{val:.3g}" for val in hour], rotation=45, ha="right") + ax.set_yticks(np.arange(n_hours)) + ax.set_yticklabels([f"{val:.3g}" for val in hour]) + + figure.tight_layout() + matplotlib.pyplot.show() + + return { + "hours": hour, + "total_fims": total_fims, + "log10_det": log10_det, + "log10_trace_inv": log10_trace_inv, + "log10_min_eig": log10_min_eig, + "log10_cond": log10_cond, + "figure": figure, + } + + +if __name__ == "__main__": + candidate_hours = np.linspace(1, 10, 50) + candidate_hours = np.concatenate([candidate_hours, [1.9321985035514362]]) + prior_information_matrix = np.array( + [[15.48181217, 357.97684273], [357.97684273, 8277.28811613]] + ) + prior_information_matrix = np.eye(2) + exp_list = [ + RooneyBieglerMultiExperiment(hour=2.1, y=8.3), + RooneyBieglerMultiExperiment(hour=10, y=10.3), + ] + from pyomo.opt import SolverFactory + + grey_box_solver = SolverFactory("cyipopt") + grey_box_solver.config.options["linear_solver"] = "ma57" + grey_box_solver.config.options['tol'] = 1e-6 + grey_box_solver.config.options['mu_strategy'] = "monotone" + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="trace", + step=1e-2, + use_grey_box_objective=True, + grey_box_solver=grey_box_solver, + grey_box_tee=True, + ) + doe.optimize_experiments() + print("Optimal experiment design:") + print(doe.results) + + scenario = doe.results["solution"]["param_scenarios"][0] + got_hours = sorted(exp["design"][0] for exp in scenario["experiments"]) + expected_hours = [1.9321985035514362, 9.999999685577139] + results = rb_multi(candidate_hours, n_exp=2, prior_FIM=prior_information_matrix) diff --git a/pyomo/contrib/doe/tests/experiment_class_example_flags.py b/pyomo/contrib/doe/tests/experiment_class_example_flags.py index c6b3cb9e0ec..33e4cf65010 100644 --- a/pyomo/contrib/doe/tests/experiment_class_example_flags.py +++ b/pyomo/contrib/doe/tests/experiment_class_example_flags.py @@ -70,3 +70,77 @@ def get_labeled_model(self): m.bad_con_2 = pyo.Constraint(expr=m.hour <= 0.0) return m + + +class RooneyBieglerMultiExperiment(RooneyBieglerExperiment): + """ + Experiment class based on the multi-experiment Rooney-Biegler prototype. + + This mirrors the implementation in + ``examples/multiexperiment-prototype/rooney_biegler_multiexperiment.py`` + while allowing test-time control over initial hour and bounds. + """ + + def __init__( + self, hour=2.0, y=10.0, theta=None, measure_error=0.1, hour_bounds=(1.0, 10.0) + ): + data = {'hour': hour, 'y': y} + super().__init__(data=data, measure_error=measure_error, theta=theta) + self.hour_bounds = hour_bounds + + def get_labeled_model(self): + m = super().get_labeled_model() + hour_lb, hour_ub = self.hour_bounds + m.hour.setlb(hour_lb) + m.hour.setub(hour_ub) + + m.sym_break_cons = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.sym_break_cons[m.hour] = None + return m + + +class RooneyBieglerMultiInputExperimentFlag(RooneyBieglerExperiment): + """ + Two-input Rooney-Biegler style experiment for symmetry-breaking tests. + + Parameters + ---------- + sym_break_flag : int + 0 -> do not add ``sym_break_cons`` suffix + 1 -> add single marker (hour) + 2 -> add multiple markers (hour and temp) + """ + + def __init__(self, hour=2.0, temp=300.0, y=10.0, sym_break_flag=1): + data = {'hour': hour, 'y': y} + super().__init__( + data=data, measure_error=0.1, theta={'asymptote': 15, 'rate_constant': 0.5} + ) + self.hour = hour + self.temp = temp + self.sym_break_flag = sym_break_flag + + def get_labeled_model(self): + m = super().get_labeled_model() + + m.hour.setlb(1.0) + m.hour.setub(10.0) + m.temp = pyo.Var(initialize=self.temp, bounds=(280.0, 340.0)) + m.temp.fix() + + # Replace base Rooney-Biegler response with two-input synthetic variant + # used only for symmetry-breaking tests. + m.del_component(m.response_function) + m.response_function = pyo.Constraint( + expr=m.y + == m.asymptote * (1 - pyo.exp(-m.rate_constant * m.hour)) + 0.01 * m.temp + ) + m.experiment_inputs[m.temp] = self.temp + + if self.sym_break_flag in (1, 2): + m.sym_break_cons = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.sym_break_cons[m.hour] = None + if self.sym_break_flag == 2: + m.sym_break_cons[m.temp] = None + + return m diff --git a/pyomo/contrib/doe/tests/test_doe_build.py b/pyomo/contrib/doe/tests/test_doe_build.py index 7ba299dc4d0..b4164c46744 100644 --- a/pyomo/contrib/doe/tests/test_doe_build.py +++ b/pyomo/contrib/doe/tests/test_doe_build.py @@ -8,6 +8,10 @@ # ____________________________________________________________________________________ import json import os.path +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch from pyomo.common.dependencies import ( numpy as np, @@ -17,20 +21,23 @@ scipy_available, ) -from pyomo.common.fileutils import this_file_dir import pyomo.common.unittest as unittest +from pyomo.contrib.doe.doe import ObjectiveLib, _DoEResultsJSONEncoder if not (numpy_available and scipy_available): raise unittest.SkipTest("Pyomo.DoE needs scipy and numpy to run tests") -if scipy_available: - from pyomo.contrib.doe import DesignOfExperiments - from pyomo.contrib.doe.examples.reactor_example import ( - ReactorExperiment as FullReactorExperiment, - ) - from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( - RooneyBieglerExperiment, - ) +from pyomo.contrib.doe import DesignOfExperiments +from pyomo.contrib.doe.examples.polynomial import PolynomialExperiment +from pyomo.contrib.doe.examples.reactor_example import ( + ReactorExperiment as FullReactorExperiment, +) +from pyomo.contrib.doe.tests.experiment_class_example_flags import ( + RooneyBieglerMultiExperiment, +) +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) from pyomo.contrib.doe.examples.rooney_biegler_doe_example import run_rooney_biegler_doe import pyomo.environ as pyo @@ -39,14 +46,6 @@ ipopt_available = SolverFactory("ipopt").available() -currdir = this_file_dir() -file_path = os.path.join(currdir, "..", "examples", "result.json") - -with open(file_path) as f: - data_ex = json.load(f) - -data_ex["control_points"] = {float(k): v for k, v in data_ex["control_points"].items()} - def get_rooney_biegler_experiment(): """Get a fresh RooneyBieglerExperiment instance for testing. @@ -97,9 +96,11 @@ def get_FIM_FIMPrior_Q_L(doe_obj=None): for i in model.output_names for j in model.parameter_names ] - sigma_inv = [1 / v for k, v in model.scenario_blocks[0].measurement_error.items()] + sigma_inv = [ + 1 / v for k, v in model.fd_scenario_blocks[0].measurement_error.items() + ] param_vals = np.array( - [[v for k, v in model.scenario_blocks[0].unknown_parameters.items()]] + [[v for k, v in model.fd_scenario_blocks[0].unknown_parameters.items()]] ) FIM_vals_np = np.array(FIM_vals).reshape((n_param, n_param)) @@ -123,7 +124,7 @@ def get_FIM_FIMPrior_Q_L(doe_obj=None): def get_standard_args(experiment, fd_method, obj_used): args = {} - args['experiment'] = experiment + args['experiment'] = None if experiment is None else [experiment] args['fd_formula'] = fd_method args['step'] = 1e-3 args['objective_option'] = obj_used @@ -152,6 +153,20 @@ def get_standard_args(experiment, fd_method, obj_used): @unittest.skipIf(not scipy_available, "scipy is not available") @unittest.skipIf(not pandas_available, "pandas is not available") class TestDoeBuild(unittest.TestCase): + def test_constructor_accepts_single_experiment_or_list(self): + # The public constructor should normalize either form into the same + # internal experiment_list representation. + single = DesignOfExperiments( + experiment=RooneyBieglerMultiExperiment(hour=2.0), + objective_option="pseudo_trace", + ) + as_list = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + ) + self.assertEqual(len(single.experiment_list), 1) + self.assertEqual(len(as_list.experiment_list), 1) + def test_rooney_biegler_fd_central_check_fd_eqns(self): fd_method = "central" obj_used = "pseudo_trace" @@ -173,16 +188,16 @@ def test_rooney_biegler_fd_central_check_fd_eqns(self): diff = (-1) ** s * doe_obj.step param_val = pyo.value( - pyo.ComponentUID(param).find_component_on(model.scenario_blocks[s]) + pyo.ComponentUID(param).find_component_on(model.fd_scenario_blocks[s]) ) - param_val_from_step = model.scenario_blocks[0].unknown_parameters[ - pyo.ComponentUID(param).find_component_on(model.scenario_blocks[0]) + param_val_from_step = model.fd_scenario_blocks[0].unknown_parameters[ + pyo.ComponentUID(param).find_component_on(model.fd_scenario_blocks[0]) ] * (1 + diff) - for k, v in model.scenario_blocks[s].unknown_parameters.items(): + for k, v in model.fd_scenario_blocks[s].unknown_parameters.items(): if pyo.ComponentUID( - k, context=model.scenario_blocks[s] + k, context=model.fd_scenario_blocks[s] ) == pyo.ComponentUID(param): continue @@ -212,17 +227,21 @@ def test_rooney_biegler_fd_backward_check_fd_eqns(self): param = model.parameter_scenarios[s] param_val = pyo.value( - pyo.ComponentUID(param).find_component_on(model.scenario_blocks[s]) + pyo.ComponentUID(param).find_component_on( + model.fd_scenario_blocks[s] + ) ) - param_val_from_step = model.scenario_blocks[0].unknown_parameters[ - pyo.ComponentUID(param).find_component_on(model.scenario_blocks[0]) + param_val_from_step = model.fd_scenario_blocks[0].unknown_parameters[ + pyo.ComponentUID(param).find_component_on( + model.fd_scenario_blocks[0] + ) ] * (1 + diff) self.assertAlmostEqual(param_val, param_val_from_step) - for k, v in model.scenario_blocks[s].unknown_parameters.items(): + for k, v in model.fd_scenario_blocks[s].unknown_parameters.items(): if (s != 0) and pyo.ComponentUID( - k, context=model.scenario_blocks[s] + k, context=model.fd_scenario_blocks[s] ) == pyo.ComponentUID(param): continue @@ -250,23 +269,56 @@ def test_rooney_biegler_fd_forward_check_fd_eqns(self): param = model.parameter_scenarios[s] param_val = pyo.value( - pyo.ComponentUID(param).find_component_on(model.scenario_blocks[s]) + pyo.ComponentUID(param).find_component_on( + model.fd_scenario_blocks[s] + ) ) - param_val_from_step = model.scenario_blocks[0].unknown_parameters[ - pyo.ComponentUID(param).find_component_on(model.scenario_blocks[0]) + param_val_from_step = model.fd_scenario_blocks[0].unknown_parameters[ + pyo.ComponentUID(param).find_component_on( + model.fd_scenario_blocks[0] + ) ] * (1 + diff) self.assertAlmostEqual(param_val, param_val_from_step) - for k, v in model.scenario_blocks[s].unknown_parameters.items(): + for k, v in model.fd_scenario_blocks[s].unknown_parameters.items(): if (s != 0) and pyo.ComponentUID( - k, context=model.scenario_blocks[s] + k, context=model.fd_scenario_blocks[s] ) == pyo.ComponentUID(param): continue other_param_val = pyo.value(k) self.assertAlmostEqual(other_param_val, v) + def test_polynomial_example_labels(self): + experiment = PolynomialExperiment() + model = experiment.get_labeled_model() + + self.assertEqual(len(model.experiment_outputs), 1) + self.assertEqual(len(model.measurement_error), 1) + self.assertEqual(len(model.experiment_inputs), 2) + self.assertEqual(len(model.unknown_parameters), 4) + + self.assertIn(model.y, model.experiment_outputs) + self.assertIn(model.y, model.measurement_error) + self.assertIn(model.x1, model.experiment_inputs) + self.assertIn(model.x2, model.experiment_inputs) + + def test_polynomial_example_create_doe_model_pynumero(self): + experiment = PolynomialExperiment() + + DoE_args = get_standard_args( + experiment, fd_method="central", obj_used="determinant" + ) + DoE_args["gradient_method"] = "pynumero" + + doe_obj = DesignOfExperiments(**DoE_args) + doe_obj.create_doe_model() + + model = doe_obj.model + self.assertEqual(len(model.scenarios), 1) + self.assertTrue(hasattr(model.fd_scenario_blocks[0], "jac_variables_wrt_param")) + def test_rooney_biegler_fd_central_design_fixing(self): fd_method = "central" obj_used = "pseudo_trace" @@ -282,7 +334,9 @@ def test_rooney_biegler_fd_central_design_fixing(self): model = doe_obj.model # Check that the design fixing constraints are generated - design_vars = [k for k, v in model.scenario_blocks[0].experiment_inputs.items()] + design_vars = [ + k for k, v in model.fd_scenario_blocks[0].experiment_inputs.items() + ] con_name_base = "global_design_eq_con_" @@ -315,7 +369,9 @@ def test_rooney_biegler_fd_backward_design_fixing(self): model = doe_obj.model # Check that the design fixing constraints are generated - design_vars = [k for k, v in model.scenario_blocks[0].experiment_inputs.items()] + design_vars = [ + k for k, v in model.fd_scenario_blocks[0].experiment_inputs.items() + ] con_name_base = "global_design_eq_con_" @@ -348,7 +404,9 @@ def test_rooney_biegler_fd_forward_design_fixing(self): model = doe_obj.model # Check that the design fixing constraints are generated - design_vars = [k for k, v in model.scenario_blocks[0].experiment_inputs.items()] + design_vars = [ + k for k, v in model.fd_scenario_blocks[0].experiment_inputs.items() + ] con_name_base = "global_design_eq_con_" @@ -502,23 +560,27 @@ def test_generate_blocks_without_model(self): doe_obj = DesignOfExperiments(**DoE_args) - doe_obj._generate_scenario_blocks() + doe_obj._generate_fd_scenario_blocks() for i in doe_obj.model.parameter_scenarios: self.assertTrue( - doe_obj.model.find_component("scenario_blocks[" + str(i) + "]") + doe_obj.model.find_component("fd_scenario_blocks[" + str(i) + "]") ) -class TestReactorExample(unittest.TestCase): - def test_reactor_update_suffix_items(self): - """Test the reactor example with updating suffix items.""" - from pyomo.contrib.doe.examples.update_suffix_doe_example import main +class TestRooneyBieglerExample(unittest.TestCase): + def test_rooney_biegler_update_suffix_items(self): + """Test updating suffix items on the lightweight Rooney-Biegler model.""" + from pyomo.contrib.parmest.utils.model_utils import update_model_from_suffix - # Run the reactor update suffix items example - suffix_obj, _, new_vals = main() + experiment = get_rooney_biegler_experiment() + model = experiment.get_labeled_model() + suffix_obj = model.measurement_error + orig_vals = np.array(list(suffix_obj.values())) + new_vals = orig_vals + 1 + + update_model_from_suffix(suffix_obj, new_vals) - # Check that the suffix object has been updated correctly for i, v in enumerate(suffix_obj.values()): self.assertAlmostEqual(v, new_vals[i], places=6) @@ -527,6 +589,18 @@ def test_reactor_update_suffix_items(self): @unittest.skipIf(not numpy_available, "Numpy is not available") @unittest.skipIf(not pandas_available, "pandas is not available") class TestDoEObjectiveOptions(unittest.TestCase): + def test_maximize_objective_set_contents(self): + # The objective-sense partition drives maximize/minimize scoring logic + # in initialization and solve helpers, so keep a direct regression on + # the public enum membership of the maximize set. + maximize = DesignOfExperiments._MAXIMIZE_OBJECTIVES + self.assertIn(ObjectiveLib.determinant, maximize) + self.assertIn(ObjectiveLib.pseudo_trace, maximize) + self.assertIn(ObjectiveLib.minimum_eigenvalue, maximize) + self.assertNotIn(ObjectiveLib.trace, maximize) + self.assertNotIn(ObjectiveLib.condition_number, maximize) + self.assertNotIn(ObjectiveLib.zero, maximize) + def test_trace_constraints(self): fd_method = "central" obj_used = "trace" @@ -569,7 +643,7 @@ def test_trace_constraints(self): for i, c in enumerate(params): for j, d in enumerate(params): - # cholesky_inv_imp: only defined for i >= j + # inverse constraint only exists for lower triangle (i >= j) if i >= j: self.assertIn( (c, d), @@ -583,7 +657,7 @@ def test_trace_constraints(self): msg=f"Unexpected cholesky_inv_cons[{c},{d}]", ) - # cholesky_LLinv_imp: only defined for i >= j + # identity constraint only defined for lower triangle (i >= j) if i >= j: self.assertIn( (c, d), @@ -640,5 +714,476 @@ def test_trace_initialization_consistency(self): self.assertAlmostEqual(val, expected, places=4) +class TestOptimizeExperimentsBuildStructure(unittest.TestCase): + """Coverage for optimize_experiments() build, output, and diagnostics behavior.""" + + def _make_solver(self): + # Make solver object with options for DoE runs. + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + return solver + + def _mock_solver_results(self, message): + return SimpleNamespace( + solver=SimpleNamespace( + status="ok", termination_condition="optimal", message=message + ) + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_init_solver_used_for_initialization_only(self): + # Tests that all pre-final solves use init_solver while the final + # optimization solve still uses the primary solver. + main_solver = self._make_solver() + init_solver = self._make_solver() + # Use distinct option values so each solver path can be identified. + main_solver.options["max_iter"] = 321 + init_solver.options["max_iter"] = 123 + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=main_solver, + ) + + # Track both how many times each solver is called and the chronological + # order of those calls. The optimize_experiments() contract is that all + # setup solves run first on init_solver and the final NLP solve runs last + # on the primary solver. + main_calls = 0 + init_calls = 0 + call_order = [] + # Record an option value on each solve so the test can verify that the + # call really went through the expected solver object, not just the + # expected phase label. + option_markers = [] + original_main_solve = main_solver.solve + original_init_solve = init_solver.solve + + def _main_solve(*args, **kwargs): + nonlocal main_calls + main_calls += 1 + call_order.append("main") + option_markers.append(main_solver.options.get("max_iter")) + return original_main_solve(*args, **kwargs) + + def _init_solve(*args, **kwargs): + nonlocal init_calls + init_calls += 1 + call_order.append("init") + option_markers.append(init_solver.options.get("max_iter")) + return original_init_solve(*args, **kwargs) + + # Patch both solver objects in place so the real solves still run while + # we collect lightweight diagnostics about solver routing. + with ( + patch.object(main_solver, "solve", side_effect=_main_solve), + patch.object(init_solver, "solve", side_effect=_init_solve), + ): + doe_obj.optimize_experiments(n_exp=2, init_solver=init_solver) + + # The exact number of initialization solves is implementation-dependent, + # but they must all occur before the one final main-solver call. + self.assertGreaterEqual(init_calls, 1) # At least one initialization solve + self.assertEqual(main_calls, 1) # Exactly one main optimization solve + self.assertEqual(call_order[-1], "main") + self.assertTrue(all(tag == "init" for tag in call_order[:-1])) + # Distinct option markers provide a second check that solver routing + # matches the expected init-versus-final phase split. + self.assertTrue(all(marker == 123 for marker in option_markers[:-1])) + self.assertEqual(option_markers[-1], 321) + # Result payloads should report the same phase-specific solver names that + # were observed through the patched solve() calls above. + self.assertEqual( + doe_obj.results["initialization"]["solver"], + getattr(init_solver, "name", str(init_solver)), + ) + self.assertEqual( + doe_obj.results["optimization_solve"]["solver"], + getattr(main_solver, "name", str(main_solver)), + ) + + def test_get_experiment_input_vars_direct_and_fd_fallback(self): + # Test the helper used by optimize_experiments() finds input vars for both + # direct models and finite-difference scenario-block models. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + ) + + model_direct = pyo.ConcreteModel() + model_direct.x = pyo.Var(initialize=2.0, bounds=(1.0, 10.0)) + model_direct.experiment_inputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + model_direct.experiment_inputs[model_direct.x] = 2.0 + vars_direct = doe_obj._get_experiment_input_vars(model_direct) + self.assertEqual([v.name for v in vars_direct], [model_direct.x.name]) + + model_fd = pyo.ConcreteModel() + model_fd.fd_scenario_blocks = pyo.Block([0]) + model_fd.fd_scenario_blocks[0].x = pyo.Var(initialize=3.0, bounds=(1.0, 10.0)) + model_fd.fd_scenario_blocks[0].experiment_inputs = pyo.Suffix( + direction=pyo.Suffix.LOCAL + ) + model_fd.fd_scenario_blocks[0].experiment_inputs[ + model_fd.fd_scenario_blocks[0].x + ] = 3.0 + vars_fd = doe_obj._get_experiment_input_vars(model_fd) + self.assertEqual( + [v.name for v in vars_fd], [model_fd.fd_scenario_blocks[0].x.name] + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_multi_experiment_structure_and_results(self): + # Test that the multi-experiment optimize_experiments() run builds the expected + # scenario/experiment structure and structured results. + solver = self._make_solver() + + doe_obj = DesignOfExperiments( + experiment=[ + RooneyBieglerMultiExperiment(hour=2.0), + RooneyBieglerMultiExperiment(hour=4.0), + ], + objective_option="pseudo_trace", + step=1e-2, + solver=solver, + ) + doe_obj.optimize_experiments() + + scenario_block = doe_obj.model.param_scenario_blocks[0] + self.assertTrue(hasattr(scenario_block, "symmetry_breaking_s0_exp1")) + self.assertEqual(len(list(scenario_block.exp_blocks.keys())), 2) + + param_scenario = doe_obj.results["solution"]["param_scenarios"][0] + self.assertEqual(doe_obj.results["initialization"]["method"], "none") + self.assertEqual( + doe_obj.results["problem"]["number_of_experiments_per_scenario"], 2 + ) + self.assertEqual(len(param_scenario["experiments"]), 2) + self.assertEqual(len(doe_obj.results["problem"]["design_variables"]), 1) + self.assertEqual(len(doe_obj.results["problem"]["parameters"]), 2) + + # Results should expose a single structured parameter-scenario payload. + self.assertIn("problem", doe_obj.results) + self.assertIn("initialization", doe_obj.results) + self.assertIn("optimization_solve", doe_obj.results) + self.assertIn("timing", doe_obj.results) + self.assertIn("solution", doe_obj.results) + self.assertEqual(doe_obj.results["solution"]["objective"], "pseudo_trace") + self.assertEqual(doe_obj.results["optimization_solve"]["status"], "ok") + self.assertFalse(doe_obj.results["problem"]["used_template_experiment"]) + self.assertEqual(len(doe_obj.results["solution"]["param_scenarios"]), 1) + self.assertEqual(param_scenario["param_scenario_id"], 0) + self.assertEqual(param_scenario["param_scenario_weight"], 1.0) + self.assertEqual(len(param_scenario["experiments"]), 2) + self.assertEqual(param_scenario["experiments"][0]["exp_id"], 0) + self.assertEqual( + doe_obj.results["problem"]["measurement_error_values"], + doe_obj.get_measurement_error_values(model=scenario_block.exp_blocks[0]), + ) + self.assertNotIn("measurement_errors", param_scenario["experiments"][0]) + + # hour of exp[0] should be <= hour of exp[1] due to symmetry breaking + h0 = param_scenario["experiments"][0]["design"][0] + h1 = param_scenario["experiments"][1]["design"][0] + self.assertLessEqual(h0, h1) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_writes_results_file(self): + # Tests that optimize_experiments() writes JSON results when given either + # a string path or a pathlib.Path for results_file. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + fd, results_path = tempfile.mkstemp(suffix=".json") + os.close(fd) + self.addCleanup( + lambda: os.path.exists(results_path) and os.remove(results_path) + ) + + doe_obj.optimize_experiments(n_exp=1, results_file=results_path) + + with open(results_path) as f: + payload = json.load(f) + self.assertEqual(payload["initialization"]["method"], "none") + self.assertTrue(payload["problem"]["used_template_experiment"]) + self.assertIn("solution", payload) + self.assertEqual(payload["solution"]["objective"], "pseudo_trace") + self.assertIn("optimization_solve", payload) + self.assertIn("problem", payload) + self.assertIn("timing", payload) + path_payload = Path(results_path) + doe_obj.optimize_experiments(n_exp=1, results_file=path_payload) + with open(results_path) as f: + payload_path = json.load(f) + self.assertEqual(payload_path["initialization"]["method"], "none") + self.assertTrue(payload_path["problem"]["used_template_experiment"]) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_single_experiment_defaults_to_template_mode(self): + # Tests that optimize_experiments() uses template mode by default when + # n_exp=1. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + doe_obj.optimize_experiments() + self.assertEqual( + doe_obj.results["problem"]["number_of_experiments_per_scenario"], 1 + ) + self.assertTrue(doe_obj.results["problem"]["used_template_experiment"]) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_zero_objective_works_without_obj_cons(self): + # The zero objective intentionally builds no scenario.obj_cons block, so + # optimize_experiments() must skip the deactivate/reactivate cycle and + # still produce the usual results payload from the square solve. + init_solver = self._make_solver() + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="zero", + step=1e-2, + solver=self._make_solver(), + ) + final_calls = {"n": 0} + + def _mock_final_solve(*args, **kwargs): + final_calls["n"] += 1 + return self._mock_solver_results("mock-zero-final") + + with patch.object(doe_obj.solver, "solve", side_effect=_mock_final_solve): + doe_obj.optimize_experiments(n_exp=2, init_solver=init_solver) + + scenario = doe_obj.model.param_scenario_blocks[0] + self.assertFalse(hasattr(scenario, "obj_cons")) + self.assertEqual(final_calls["n"], 1) + self.assertEqual(doe_obj.results["optimization_solve"]["status"], "ok") + self.assertEqual( + doe_obj.results["optimization_solve"]["message"], "mock-zero-final" + ) + self.assertEqual( + len(doe_obj.results["solution"]["param_scenarios"][0]["experiments"]), 2 + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_trace_roundoff_flag_builds_extra_constraints(self): + # The multi-experiment trace path optionally adds extra Cholesky/FIM + # diagonal constraints to reduce roundoff drift. Keep the square-solve + # build real, but mock the final NLP solve because this test is only + # checking that the additional constraints were created. + init_solver = self._make_solver() + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="trace", + step=1e-2, + solver=self._make_solver(), + improve_cholesky_roundoff_error=True, + ) + final_calls = {"n": 0} + + def _mock_final_solve(*args, **kwargs): + final_calls["n"] += 1 + return self._mock_solver_results("mock-trace-roundoff-final") + + with patch.object(doe_obj.solver, "solve", side_effect=_mock_final_solve): + doe_obj.optimize_experiments(n_exp=2, init_solver=init_solver) + + scenario = doe_obj.model.param_scenario_blocks[0] + parameter_names = list(scenario.exp_blocks[0].parameter_names) + + self.assertEqual(final_calls["n"], 1) + self.assertTrue(hasattr(scenario.obj_cons, "cholesky_fim_diag_cons")) + self.assertTrue(hasattr(scenario.obj_cons, "cholesky_fim_inv_diag_cons")) + self.assertEqual( + len(scenario.obj_cons.cholesky_fim_diag_cons), len(parameter_names) ** 2 + ) + self.assertEqual( + len(scenario.obj_cons.cholesky_fim_inv_diag_cons), len(parameter_names) ** 2 + ) + self.assertEqual( + doe_obj.results["optimization_solve"]["message"], + "mock-trace-roundoff-final", + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_timing_includes_lhs_phase_separately(self): + # Tests that LHS initialization timing is tracked separately and + # contributes additively to total runtime accounting. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + doe_obj.optimize_experiments( + n_exp=2, init_method="lhs", init_n_samples=2, init_seed=11 + ) + + timing = doe_obj.results["timing"] + self.assertIn("lhs_initialization_time_s", timing) + self.assertGreaterEqual(timing["lhs_initialization_time_s"], 0.0) + self.assertAlmostEqual( + timing["total_time_s"], + timing["build_time_s"] + + timing["lhs_initialization_time_s"] + + timing["initialization_time_s"] + + timing["optimization_solve_time_s"], + places=8, + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_symmetry_log_once_per_scenario(self): + # Tests that symmetry-breaking constraint logging is emitted once per + # scenario (not once per generated constraint). + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + with self.assertLogs("pyomo.contrib.doe.doe", level="INFO") as log_cm: + doe_obj.optimize_experiments(n_exp=3) + + matching = [ + m + for m in log_cm.output + if "Added 2 symmetry breaking constraints for scenario 0" in m + ] + self.assertEqual(len(matching), 1) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_lhs_diagnostics_populated(self): + # Tests that threaded LHS initialization records diagnostics needed for + # debugging and performance visibility. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + doe_obj.optimize_experiments( + n_exp=2, + init_method="lhs", + init_n_samples=2, + init_seed=11, + init_parallel=True, + init_combo_parallel=True, + init_n_workers=2, + init_combo_chunk_size=2, + init_combo_parallel_threshold=1, + init_max_wall_clock_time=60.0, + ) + lhs_init = doe_obj.results["initialization"] + self.assertEqual(lhs_init["candidate_fim_evaluation_mode"], "thread") + self.assertEqual(lhs_init["combination_scoring_mode"], "thread") + self.assertEqual(lhs_init["workers"], 2) + self.assertFalse(lhs_init["timed_out"]) + self.assertGreater(lhs_init["time_s"], 0.0) + self.assertIn("best_initial_objective_value", lhs_init) + self.assertIsInstance(lhs_init["best_initial_objective_value"], float) + self.assertTrue(np.isfinite(lhs_init["best_initial_objective_value"])) + self.assertGreater(lhs_init["best_initial_objective_value"], 0.0) + self.assertIn("best_initial_objective_value_log10", lhs_init) + self.assertIsInstance(lhs_init["best_initial_objective_value_log10"], float) + self.assertAlmostEqual( + lhs_init["best_initial_objective_value_log10"], + np.log10(lhs_init["best_initial_objective_value"]), + places=12, + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_termination_message_bytes(self): + # Tests that solver termination messages returned as bytes are decoded + # and persisted as plain strings in results. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + original_solve = doe_obj.solver.solve + + def _solve_with_bytes_message(*args, **kwargs): + res = original_solve(*args, **kwargs) + res.solver.message = b"byte-message" + return res + + with patch.object( + doe_obj.solver, "solve", side_effect=_solve_with_bytes_message + ): + doe_obj.optimize_experiments(n_exp=1) + + self.assertEqual( + doe_obj.results["optimization_solve"]["message"], "byte-message" + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_optimize_experiments_termination_message_fallback_to_str(self): + # Tests that non-string termination messages fall back to str(message) so + # results always store a serializable user-facing value. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + original_solve = doe_obj.solver.solve + + class _CustomMessage: + def __str__(self): + return "custom-message" + + def _solve_with_custom_message(*args, **kwargs): + res = original_solve(*args, **kwargs) + res.solver.message = _CustomMessage() + return res + + with patch.object( + doe_obj.solver, "solve", side_effect=_solve_with_custom_message + ): + doe_obj.optimize_experiments(n_exp=1) + + self.assertEqual( + doe_obj.results["optimization_solve"]["message"], "custom-message" + ) + + +class TestDoeResultsSerialization(unittest.TestCase): + """Coverage for DoE results payload serialization helpers.""" + + def test_doe_results_json_encoder_handles_numpy_and_enum(self): + # Results payloads are written to JSON from optimize_experiments(), so + # the encoder must normalize numpy scalars/arrays and ObjectiveLib enums. + payload = { + "scalar": np.int64(7), + "array": np.array([1.0, 2.0]), + "objective": ObjectiveLib.trace, + } + encoded = json.dumps(payload, cls=_DoEResultsJSONEncoder) + decoded = json.loads(encoded) + + self.assertEqual(decoded["scalar"], 7) + self.assertEqual(decoded["array"], [1.0, 2.0]) + self.assertEqual(decoded["objective"], str(ObjectiveLib.trace)) + + +class TestDoeBuildHelpers(unittest.TestCase): + """Coverage for small matrix helpers used by build/solve paths.""" + + def test_symmetrize_lower_tri_helper(self): + # Only the lower-triangular FIM is stored in several code paths; this + # helper must mirror it to a full symmetric matrix without doubling the diagonal. + m = np.array([[1.0, 0.0, 0.0], [2.0, 3.0, 0.0], [4.0, 5.0, 6.0]]) + got = DesignOfExperiments._symmetrize_lower_tri(m) + expected = np.array([[1.0, 2.0, 4.0], [2.0, 3.0, 5.0], [4.0, 5.0, 6.0]]) + self.assertTrue(np.allclose(got, expected, atol=1e-12)) + + if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/doe/tests/test_doe_errors.py b/pyomo/contrib/doe/tests/test_doe_errors.py index 10165aa6267..6d34e6ccd7d 100644 --- a/pyomo/contrib/doe/tests/test_doe_errors.py +++ b/pyomo/contrib/doe/tests/test_doe_errors.py @@ -6,7 +6,8 @@ # Solutions of Sandia, LLC, the U.S. Government retains certain rights in this # software. This software is distributed under the 3-clause BSD License. # ____________________________________________________________________________________ - +import json +import warnings from pyomo.common.dependencies import ( numpy as np, numpy_available, @@ -17,26 +18,36 @@ from pyomo.common.errors import DeveloperError import pyomo.common.unittest as unittest +from unittest.mock import patch if not (numpy_available and scipy_available): raise unittest.SkipTest("Pyomo.DoE needs scipy and numpy to run tests") -if scipy_available: - from pyomo.contrib.doe import DesignOfExperiments - from pyomo.contrib.doe.tests.experiment_class_example_flags import ( - BadExperiment, - RooneyBieglerExperimentFlag, - ) - from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( - RooneyBieglerExperiment, - ) +from pyomo.contrib.doe import DesignOfExperiments +from pyomo.contrib.doe.doe import InitializationMethod, _DoEResultsJSONEncoder +from pyomo.contrib.doe.tests.experiment_class_example_flags import ( + BadExperiment, + RooneyBieglerExperimentFlag, + RooneyBieglerMultiExperiment, + RooneyBieglerMultiInputExperimentFlag, +) +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) +from pyomo.contrib.doe.examples.polynomial import PolynomialExperiment from pyomo.contrib.doe.examples.rooney_biegler_doe_example import run_rooney_biegler_doe +import pyomo.environ as pyo from pyomo.opt import SolverFactory ipopt_available = SolverFactory("ipopt").available() +class _DummyExperiment: + def get_labeled_model(self, **kwargs): + raise RuntimeError("Should not be called in argument-validation tests") + + def get_rooney_biegler_experiment_flag(): """Get a fresh RooneyBieglerExperimentFlag instance for testing. @@ -53,9 +64,17 @@ def get_rooney_biegler_experiment_flag(): ) +def _make_ipopt_solver(): + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + return solver + + def get_standard_args(experiment, fd_method, obj_used, flag): args = {} - args['experiment'] = experiment + args['experiment'] = None if experiment is None else [experiment] args['fd_formula'] = fd_method args['step'] = 1e-3 args['objective_option'] = obj_used @@ -84,20 +103,36 @@ def get_standard_args(experiment, fd_method, obj_used, flag): @unittest.skipIf(not scipy_available, "scipy is not available") @unittest.skipIf(not pandas_available, "pandas is not available") class TestDoEErrors(unittest.TestCase): + def _make_dummy_optimize_experiments_doe(self, n_experiments=1): + return DesignOfExperiments( + experiment=[_DummyExperiment() for _ in range(n_experiments)], + objective_option="pseudo_trace", + ) + def test_experiment_none_error(self): fd_method = "central" obj_used = "pseudo_trace" flag_val = 1 # Value for faulty model build mode - 1: No exp outputs with self.assertRaisesRegex( - ValueError, "Experiment object must be provided to perform DoE." + ValueError, "The 'experiment' parameter is required" ): # Experiment provided as None DoE_args = get_standard_args(None, fd_method, obj_used, flag_val) doe_obj = DesignOfExperiments(**DoE_args) - def test_reactor_check_no_get_labeled_model(self): + def test_experiment_empty_list_error(self): + with self.assertRaisesRegex( + ValueError, "The 'experiment' list cannot be empty" + ): + DesignOfExperiments(experiment=[], objective_option="pseudo_trace") + + def test_doe_results_json_encoder_unsupported_object_raises(self): + with self.assertRaises(TypeError): + json.dumps({"x": object()}, cls=_DoEResultsJSONEncoder) + + def test_bad_experiment_check_no_get_labeled_model(self): fd_method = "central" obj_used = "pseudo_trace" flag_val = 1 # Value for faulty model build mode - 1: No exp outputs @@ -105,14 +140,13 @@ def test_reactor_check_no_get_labeled_model(self): experiment = BadExperiment() with self.assertRaisesRegex( - ValueError, - "The experiment object must have a ``get_labeled_model`` function", + ValueError, "Experiment at index .* must have a.*get_labeled_model" ): DoE_args = get_standard_args(experiment, fd_method, obj_used, flag_val) doe_obj = DesignOfExperiments(**DoE_args) - def test_reactor_check_no_experiment_outputs(self): + def test_rooney_biegler_check_no_experiment_outputs(self): fd_method = "central" obj_used = "pseudo_trace" flag_val = 1 # Value for faulty model build mode - 1: No exp outputs @@ -129,7 +163,7 @@ def test_reactor_check_no_experiment_outputs(self): ): doe_obj.create_doe_model() - def test_reactor_check_no_measurement_error(self): + def test_rooney_biegler_check_no_measurement_error(self): fd_method = "central" obj_used = "pseudo_trace" flag_val = 2 # Value for faulty model build mode - 2: No meas error @@ -146,7 +180,7 @@ def test_reactor_check_no_measurement_error(self): ): doe_obj.create_doe_model() - def test_reactor_check_no_experiment_inputs(self): + def test_rooney_biegler_check_no_experiment_inputs(self): fd_method = "central" obj_used = "pseudo_trace" flag_val = 3 # Value for faulty model build mode - 3: No exp inputs/design vars @@ -163,7 +197,7 @@ def test_reactor_check_no_experiment_inputs(self): ): doe_obj.create_doe_model() - def test_reactor_check_no_unknown_parameters(self): + def test_rooney_biegler_check_no_unknown_parameters(self): fd_method = "central" obj_used = "pseudo_trace" flag_val = 4 # Value for faulty model build mode - 4: No unknown params @@ -180,7 +214,7 @@ def test_reactor_check_no_unknown_parameters(self): ): doe_obj.create_doe_model() - def test_reactor_check_bad_prior_size(self): + def test_rooney_biegler_check_bad_prior_size(self): fd_method = "central" obj_used = "pseudo_trace" @@ -202,7 +236,7 @@ def test_reactor_check_bad_prior_size(self): ): doe_obj.create_doe_model() - def test_reactor_check_bad_prior_negative_eigenvalue(self): + def test_rooney_biegler_check_bad_prior_negative_eigenvalue(self): from pyomo.contrib.doe.doe import _SMALL_TOLERANCE_DEFINITENESS fd_method = "central" @@ -226,7 +260,7 @@ def test_reactor_check_bad_prior_negative_eigenvalue(self): ): doe_obj.create_doe_model() - def test_reactor_check_bad_prior_not_symmetric(self): + def test_rooney_biegler_check_bad_prior_not_symmetric(self): from pyomo.contrib.doe.utils import _SMALL_TOLERANCE_SYMMETRY fd_method = "central" @@ -250,7 +284,7 @@ def test_reactor_check_bad_prior_not_symmetric(self): ): doe_obj.create_doe_model() - def test_reactor_check_bad_jacobian_init_size(self): + def test_rooney_biegler_check_bad_jacobian_init_size(self): fd_method = "central" obj_used = "pseudo_trace" @@ -271,7 +305,7 @@ def test_reactor_check_bad_jacobian_init_size(self): ): doe_obj.create_doe_model() - def test_reactor_check_unbuilt_update_FIM(self): + def test_rooney_biegler_check_unbuilt_update_FIM(self): fd_method = "central" obj_used = "pseudo_trace" @@ -290,7 +324,7 @@ def test_reactor_check_unbuilt_update_FIM(self): ): doe_obj.update_FIM_prior(FIM=FIM_update) - def test_reactor_check_none_update_FIM(self): + def test_rooney_biegler_check_none_update_FIM(self): fd_method = "central" obj_used = "pseudo_trace" @@ -308,7 +342,7 @@ def test_reactor_check_none_update_FIM(self): ): doe_obj.update_FIM_prior(FIM=FIM_update) - def test_reactor_check_results_file_name(self): + def test_rooney_biegler_check_results_file_name(self): fd_method = "central" obj_used = "pseudo_trace" @@ -323,7 +357,7 @@ def test_reactor_check_results_file_name(self): ): doe_obj.run_doe(results_file=int(15)) - def test_reactor_check_measurement_and_output_length_match(self): + def test_rooney_biegler_check_measurement_and_output_length_match(self): fd_method = "central" obj_used = "pseudo_trace" flag_val = ( @@ -344,7 +378,7 @@ def test_reactor_check_measurement_and_output_length_match(self): doe_obj.create_doe_model() @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") - def test_reactor_grid_search_des_range_inputs(self): + def test_rooney_biegler_grid_search_des_range_inputs(self): fd_method = "central" obj_used = "determinant" @@ -365,7 +399,7 @@ def test_reactor_grid_search_des_range_inputs(self): ) @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") - def test_reactor_premature_figure_drawing(self): + def test_rooney_biegler_premature_figure_drawing(self): fd_method = "central" obj_used = "determinant" @@ -383,7 +417,7 @@ def test_reactor_premature_figure_drawing(self): doe_obj.draw_factorial_figure() @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") - def test_reactor_figure_drawing_no_des_var_names(self): + def test_rooney_biegler_figure_drawing_no_des_var_names(self): fd_method = "central" obj_used = "determinant" @@ -407,7 +441,7 @@ def test_reactor_figure_drawing_no_des_var_names(self): doe_obj.draw_factorial_figure(results=doe_obj.fim_factorial_results) @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") - def test_reactor_figure_drawing_no_sens_names(self): + def test_rooney_biegler_figure_drawing_no_sens_names(self): fd_method = "central" obj_used = "determinant" @@ -429,7 +463,7 @@ def test_reactor_figure_drawing_no_sens_names(self): doe_obj.draw_factorial_figure() @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") - def test_reactor_figure_drawing_no_fixed_names(self): + def test_rooney_biegler_figure_drawing_no_fixed_names(self): fd_method = "central" obj_used = "determinant" @@ -451,7 +485,7 @@ def test_reactor_figure_drawing_no_fixed_names(self): doe_obj.draw_factorial_figure(sensitivity_design_variables={"dummy": "var"}) @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") - def test_reactor_figure_drawing_bad_fixed_names(self): + def test_rooney_biegler_figure_drawing_bad_fixed_names(self): fd_method = "central" obj_used = "determinant" @@ -477,7 +511,7 @@ def test_reactor_figure_drawing_bad_fixed_names(self): ) @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") - def test_reactor_figure_drawing_bad_sens_names(self): + def test_rooney_biegler_figure_drawing_bad_sens_names(self): fd_method = "central" obj_used = "determinant" @@ -503,7 +537,91 @@ def test_reactor_figure_drawing_bad_sens_names(self): fixed_design_variables={"hour": 1}, ) - def test_reactor_check_get_FIM_without_FIM(self): + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_polynomial_figure_drawing_more_than_two_sens_vars(self): + fd_method = "central" + obj_used = "determinant" + + experiment = PolynomialExperiment() + + DoE_args = get_standard_args(experiment, fd_method, obj_used, flag=None) + DoE_args["gradient_method"] = "pynumero" + DoE_args["scale_nominal_param_value"] = False + + doe_obj = DesignOfExperiments(**DoE_args) + + synthetic_results = { + "x1": [0.0, 2.5], + "x2": [0.0, 0.0], + "x3": [0.0, 0.0], + "log10 D-opt": [1.0, 2.0], + "log10 A-opt": [0.1, 0.2], + "log10 pseudo A-opt": [0.3, 0.4], + "log10 E-opt": [0.5, 0.6], + "log10 ME-opt": [0.7, 0.8], + "eigval_min": [1.0, 2.0], + "eigval_max": [3.0, 4.0], + "det_FIM": [5.0, 6.0], + "trace_cov": [7.0, 8.0], + "trace_FIM": [9.0, 10.0], + "solve_time": [0.01, 0.02], + } + + with self.assertRaisesRegex( + NotImplementedError, + "Currently, only 1D and 2D sensitivity plotting is supported.", + ): + doe_obj.draw_factorial_figure( + results=synthetic_results, + sensitivity_design_variables=["x1", "x2", "x3"], + fixed_design_variables={}, + full_design_variable_names=["x1", "x2", "x3"], + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_polynomial_figure_drawing_requires_all_other_design_vars_fixed(self): + fd_method = "central" + obj_used = "determinant" + + experiment = PolynomialExperiment() + + DoE_args = get_standard_args(experiment, fd_method, obj_used, flag=None) + DoE_args["gradient_method"] = "pynumero" + DoE_args["scale_nominal_param_value"] = False + + doe_obj = DesignOfExperiments(**DoE_args) + + # Use a synthetic table shape that mimics multiple design variables so we can + # exercise the dimensionality guard without needing a heavier example. + synthetic_results = { + "x1": [0.0, 2.5], + "x2": [0.0, 0.0], + "x3": [0.0, 0.0], + "log10 D-opt": [1.0, 2.0], + "log10 A-opt": [0.1, 0.2], + "log10 pseudo A-opt": [0.3, 0.4], + "log10 E-opt": [0.5, 0.6], + "log10 ME-opt": [0.7, 0.8], + "eigval_min": [1.0, 2.0], + "eigval_max": [3.0, 4.0], + "det_FIM": [5.0, 6.0], + "trace_cov": [7.0, 8.0], + "trace_FIM": [9.0, 10.0], + "solve_time": [0.01, 0.02], + } + + with self.assertRaisesRegex( + ValueError, + "Error: All design variables that are not used to generate sensitivity plots must be fixed.", + ): + doe_obj.draw_factorial_figure( + results=synthetic_results, + sensitivity_design_variables=["x1"], + fixed_design_variables={"x2": 0.0}, + full_design_variable_names=["x1", "x2", "x3"], + ) + + def test_rooney_biegler_check_get_FIM_without_FIM(self): fd_method = "central" obj_used = "pseudo_trace" @@ -520,7 +638,7 @@ def test_reactor_check_get_FIM_without_FIM(self): ): doe_obj.get_FIM() - def test_reactor_check_get_sens_mat_without_model(self): + def test_rooney_biegler_check_get_sens_mat_without_model(self): fd_method = "central" obj_used = "pseudo_trace" @@ -538,7 +656,7 @@ def test_reactor_check_get_sens_mat_without_model(self): ): doe_obj.get_sensitivity_matrix() - def test_reactor_check_get_exp_inputs_without_model(self): + def test_rooney_biegler_check_get_exp_inputs_without_model(self): fd_method = "central" obj_used = "pseudo_trace" @@ -556,7 +674,7 @@ def test_reactor_check_get_exp_inputs_without_model(self): ): doe_obj.get_experiment_input_values() - def test_reactor_check_get_exp_outputs_without_model(self): + def test_rooney_biegler_check_get_exp_outputs_without_model(self): fd_method = "central" obj_used = "pseudo_trace" @@ -574,7 +692,7 @@ def test_reactor_check_get_exp_outputs_without_model(self): ): doe_obj.get_experiment_output_values() - def test_reactor_check_get_unknown_params_without_model(self): + def test_rooney_biegler_check_get_unknown_params_without_model(self): fd_method = "central" obj_used = "pseudo_trace" @@ -592,7 +710,7 @@ def test_reactor_check_get_unknown_params_without_model(self): ): doe_obj.get_unknown_parameter_values() - def test_reactor_check_get_meas_error_without_model(self): + def test_rooney_biegler_check_get_meas_error_without_model(self): fd_method = "central" obj_used = "pseudo_trace" @@ -677,7 +795,7 @@ def test_bad_FD_generate_scens(self): "Please report this to the Pyomo Developers.", ): doe_obj.fd_formula = "bad things" - doe_obj._generate_scenario_blocks() + doe_obj._generate_fd_scenario_blocks() @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") def test_bad_FD_seq_compute_FIM(self): @@ -762,6 +880,50 @@ def test_bad_compute_FIM_option(self): ): doe_obj.compute_FIM(method="Bad Method") + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_run_doe_rejects_kaug_gradient_method(self): + experiment = PolynomialExperiment() + + DoE_args = get_standard_args( + experiment, fd_method="central", obj_used="pseudo_trace", flag=None + ) + DoE_args["gradient_method"] = "kaug" + DoE_args["scale_nominal_param_value"] = False + + doe_obj = DesignOfExperiments(**DoE_args) + + with self.assertRaisesRegex( + ValueError, "Cannot use GradientMethod.kaug for DoE optimization." + ): + doe_obj.run_doe() + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_compute_FIM_multi_experiment_parameter_value_mismatch(self): + fd_method = "central" + obj_used = "pseudo_trace" + + DoE_args = get_standard_args( + RooneyBieglerMultiExperiment(hour=1.5, y=9.0), fd_method, obj_used, None + ) + DoE_args["experiment"] = [ + RooneyBieglerMultiExperiment( + hour=1.5, y=9.0, theta={'asymptote': 15, 'rate_constant': 0.5} + ), + RooneyBieglerMultiExperiment( + hour=3.5, y=12.0, theta={'asymptote': 16, 'rate_constant': 0.5} + ), + ] + doe_obj = DesignOfExperiments(**DoE_args) + + def _fake_sequential(*args, **kwargs): + doe_obj.seq_FIM = np.eye(2) + + with patch.object(doe_obj, "_sequential_FIM", side_effect=_fake_sequential): + with self.assertRaisesRegex( + ValueError, "must share the same unknown parameter values" + ): + doe_obj.compute_FIM(method="sequential") + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") def test_invalid_trace_without_cholesky(self): fd_method = "central" @@ -781,6 +943,482 @@ def test_invalid_trace_without_cholesky(self): ): doe_obj.create_objective_function() + def test_optimize_experiments_init_argument_validation_cases(self): + # These argument checks all fail before any model build, so a single + # table-driven test keeps the user-facing validation contracts aligned + # without repeating the same dummy DoE setup for each branch. + cases = [ + ( + "unsupported init_method", + {"init_method": "bad"}, + ValueError, + r"``init_method`` must be one of \[None, 'lhs'\], got 'bad'.", + ), + ( + "enum init_method still validates init_n_samples", + {"init_method": InitializationMethod.lhs, "init_n_samples": 0}, + ValueError, + r"``init_n_samples`` must be a positive integer, got 0.", + ), + ( + "non-positive init_n_samples", + {"init_method": "lhs", "init_n_samples": 0}, + ValueError, + r"``init_n_samples`` must be a positive integer, got 0.", + ), + ( + "non-integer init_n_samples", + {"init_method": "lhs", "init_n_samples": 2.5}, + ValueError, + r"``init_n_samples`` must be a positive integer, got 2.5.", + ), + ( + "init_parallel must be bool", + {"init_method": "lhs", "init_parallel": 1}, + ValueError, + r"``init_parallel`` must be a bool, got 1.", + ), + ( + "init_combo_parallel must be bool", + {"init_method": "lhs", "init_combo_parallel": "yes"}, + ValueError, + r"``init_combo_parallel`` must be a bool", + ), + ( + "init_n_workers must be positive integer", + {"init_method": "lhs", "init_n_workers": 0}, + ValueError, + r"``init_n_workers`` must be None or a positive integer", + ), + ( + "init_combo_chunk_size must be positive integer", + {"init_method": "lhs", "init_combo_chunk_size": 0}, + ValueError, + r"``init_combo_chunk_size`` must be a positive integer", + ), + ( + "init_combo_parallel_threshold must be positive integer", + {"init_method": "lhs", "init_combo_parallel_threshold": 0}, + ValueError, + r"``init_combo_parallel_threshold`` must be a positive integer", + ), + ( + "init_max_wall_clock_time must be positive", + {"init_method": "lhs", "init_max_wall_clock_time": 0}, + ValueError, + r"``init_max_wall_clock_time`` must be None or a positive number", + ), + ( + "init_max_wall_clock_time rejects nan", + {"init_method": "lhs", "init_max_wall_clock_time": float("nan")}, + ValueError, + r"``init_max_wall_clock_time`` must be None or a positive number", + ), + ( + "init_max_wall_clock_time rejects inf", + {"init_method": "lhs", "init_max_wall_clock_time": float("inf")}, + ValueError, + r"``init_max_wall_clock_time`` must be None or a positive number", + ), + ( + "init_seed must be integer", + { + "n_exp": 2, + "init_method": "lhs", + "init_n_samples": 2, + "init_seed": 1.5, + }, + ValueError, + r"``init_seed`` must be None or an integer", + ), + ] + + for label, kwargs, exc_type, regex in cases: + with self.subTest(case=label): + doe_obj = self._make_dummy_optimize_experiments_doe() + with self.assertRaisesRegex(exc_type, regex): + doe_obj.optimize_experiments(**kwargs) + + def test_optimize_experiments_lhs_requires_template_mode(self): + # Tests that LHS initialization is disallowed in user-initialized multi-experiment mode. + doe_obj = DesignOfExperiments( + experiment=[_DummyExperiment(), _DummyExperiment()], + objective_option="pseudo_trace", + ) + with self.assertRaisesRegex( + ValueError, + r"``init_method='lhs'`` is currently supported only in template mode", + ): + doe_obj.optimize_experiments(init_method="lhs") + + def test_optimize_experiments_lhs_requires_scipy(self): + # Tests that LHS initialization requires scipy to be available. + doe_obj = DesignOfExperiments( + experiment=[_DummyExperiment()], objective_option="pseudo_trace" + ) + with patch("pyomo.contrib.doe.doe.scipy_available", False): + with self.assertRaisesRegex( + ImportError, r"LHS initialization requires scipy" + ): + doe_obj.optimize_experiments(init_method="lhs") + + def test_optimize_experiments_general_argument_validation_cases(self): + # These are the remaining lightweight API validations whose only + # contract is the error raised for a bad user-facing kwarg/value pair. + cases = [ + ( + "n_exp disallowed with multi-experiment list", + 2, + {"n_exp": 2}, + ValueError, + r"``n_exp`` must not be set when the experiment list contains more than one experiment", + ), + ( + "n_exp must be positive", + 1, + {"n_exp": 0}, + ValueError, + r"``n_exp`` must be a positive integer, got 0.", + ), + ( + "results_file must be str or Path", + 1, + {"results_file": 5}, + ValueError, + r"``results_file`` must be either a Path object or a string.", + ), + ( + "init_solver must have solve", + 1, + {"init_solver": object()}, + ValueError, + r"``init_solver`` must be None or a solver object with a 'solve' method.", + ), + ] + + for label, n_experiments, kwargs, exc_type, regex in cases: + with self.subTest(case=label): + doe_obj = self._make_dummy_optimize_experiments_doe( + n_experiments=n_experiments + ) + with self.assertRaisesRegex(exc_type, regex): + doe_obj.optimize_experiments(**kwargs) + + +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") +@unittest.skipIf(not numpy_available, "Numpy is not available") +@unittest.skipIf(not scipy_available, "scipy is not available") +@unittest.skipIf(not pandas_available, "pandas is not available") +class TestDoEErrorsRequiringSolver(unittest.TestCase): + def _make_solver(self): + return _make_ipopt_solver() + + def test_optimize_experiments_non_greybox_rejects_e_and_me_objectives(self): + # E-opt and ME-opt require the greybox objective path in + # optimize_experiments(); the standard algebraic multi-experiment build + # should fail fast with a user-facing validation error instead. + for objective_option in ("minimum_eigenvalue", "condition_number"): + with self.subTest(objective=objective_option): + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0, y=10.0)], + objective_option=objective_option, + step=1e-2, + solver=self._make_solver(), + ) + + with self.assertRaisesRegex( + ValueError, + rf"objective_option='{objective_option}' requires " + r"use_grey_box_objective=True\.", + ): + doe_obj.optimize_experiments(n_exp=2) + + def test_optimize_experiments_trace_requires_cholesky_or_greybox(self): + # Multi-experiment trace uses the Cholesky-based build unless the + # greybox objective path is enabled, so optimize_experiments() should + # reject ``_Cholesky_option=False`` before any solve phase begins. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0, y=10.0)], + objective_option="trace", + step=1e-2, + solver=self._make_solver(), + _Cholesky_option=False, + ) + + with self.assertRaisesRegex( + ValueError, + r"objective_option='trace' currently only implemented with " + r"``_Cholesky_option=True`` or ``use_grey_box_objective=True``\.", + ): + doe_obj.optimize_experiments(n_exp=2) + + def test_optimize_experiments_requires_matching_unknown_parameter_values(self): + # Tests that user-initialized multi-experiment mode rejects experiments + # that linearize around different nominal theta values. + doe_obj = DesignOfExperiments( + experiment=[ + RooneyBieglerMultiExperiment( + hour=2.0, y=10.0, theta={'asymptote': 15.0, 'rate_constant': 0.5} + ), + RooneyBieglerMultiExperiment( + hour=3.0, y=11.0, theta={'asymptote': 15.0, 'rate_constant': 0.6} + ), + ], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + + with self.assertRaisesRegex( + ValueError, "must use the same nominal values for unknown_parameters" + ): + doe_obj.optimize_experiments() + + def test_optimize_experiments_requires_matching_unknown_parameter_labels(self): + # Tests that user-initialized multi-experiment mode rejects experiments + # whose unknown-parameter sets differ. + class _DifferentUnknownParameterExperiment: + def __init__(self, base_exp): + self._base_exp = base_exp + + def get_labeled_model(self, **kwargs): + m = self._base_exp.get_labeled_model(**kwargs) + m.fake_theta = pyo.Var(initialize=1.0) + m.fake_theta.fix() + m.del_component(m.unknown_parameters) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + [ + (m.asymptote, pyo.value(m.asymptote)), + (m.fake_theta, pyo.value(m.fake_theta)), + ] + ) + return m + + doe_obj = DesignOfExperiments( + experiment=[ + RooneyBieglerMultiExperiment(hour=2.0, y=10.0), + _DifferentUnknownParameterExperiment( + RooneyBieglerMultiExperiment(hour=3.0, y=11.0) + ), + ], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + + with self.assertRaisesRegex( + ValueError, "must define the same unknown_parameters in the same order" + ): + doe_obj.optimize_experiments() + + def test_optimize_experiments_requires_matching_unknown_parameter_order(self): + # Tests that user-initialized multi-experiment mode rejects experiments + # whose unknown parameters appear in a different order. + class _ReorderedUnknownParameterExperiment: + def __init__(self, base_exp): + self._base_exp = base_exp + + def get_labeled_model(self, **kwargs): + m = self._base_exp.get_labeled_model(**kwargs) + m.del_component(m.unknown_parameters) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + [ + (m.rate_constant, pyo.value(m.rate_constant)), + (m.asymptote, pyo.value(m.asymptote)), + ] + ) + return m + + doe_obj = DesignOfExperiments( + experiment=[ + RooneyBieglerMultiExperiment(hour=2.0, y=10.0), + _ReorderedUnknownParameterExperiment( + RooneyBieglerMultiExperiment(hour=3.0, y=11.0) + ), + ], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + + with self.assertRaisesRegex( + ValueError, "must define the same unknown_parameters in the same order" + ): + doe_obj.optimize_experiments() + + def test_optimize_experiments_sym_break_var_must_be_input(self): + # Tests that symmetry-breaking marker variables must also be experiment inputs. + class _BadSymBreakExperiment: + def __init__(self, base_exp): + self._base_exp = base_exp + + def get_labeled_model(self, **kwargs): + m = self._base_exp.get_labeled_model(**kwargs) + m.sym_break_cons = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.sym_break_cons[next(iter(m.unknown_parameters.keys()))] = None + return m + + exp = _BadSymBreakExperiment(RooneyBieglerMultiExperiment(hour=2.0, y=10.0)) + doe_obj = DesignOfExperiments( + experiment=[exp], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + + with self.assertRaisesRegex( + ValueError, "sym_break_cons.*must also be an experiment input variable" + ): + doe_obj.optimize_experiments(n_exp=2) + + def test_optimize_experiments_symmetry_mapping_failure_raises(self): + # Tests that failure to map symmetry variable across experiment blocks raises. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="pseudo_trace", + step=1e-2, + ) + probe_model = doe_obj.experiment_list[0].get_labeled_model( + **doe_obj.get_labeled_model_args + ) + sym_var_name = next(iter(probe_model.experiment_inputs.keys())).local_name + original_find = pyo.ComponentUID.find_component_on + + def _fail_only_symmetry_mapping(cuid, block): + if ( + sym_var_name in str(cuid) + and hasattr(block, "experiment_inputs") + and block.index() == 0 + ): + return None + return original_find(cuid, block) + + with patch( + "pyomo.contrib.doe.doe.pyo.ComponentUID.find_component_on", + autospec=True, + side_effect=_fail_only_symmetry_mapping, + ): + with self.assertRaisesRegex( + RuntimeError, "Failed to map symmetry breaking variable" + ): + doe_obj.optimize_experiments(n_exp=2) + + def test_optimize_experiments_symmetry_breaking_default_variable_warning(self): + # Tests that missing explicit symmetry marker triggers warning and default choice. + doe_obj = DesignOfExperiments( + experiment=[ + RooneyBieglerMultiInputExperimentFlag(hour=2.0, sym_break_flag=0), + RooneyBieglerMultiInputExperimentFlag(hour=4.0, sym_break_flag=0), + ], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + with self.assertLogs("pyomo.contrib.doe.doe", level="WARNING") as cm: + doe_obj.optimize_experiments() + self.assertTrue( + any("No symmetry breaking variable specified" in msg for msg in cm.output) + ) + self.assertTrue( + hasattr(doe_obj.model.param_scenario_blocks[0], "symmetry_breaking_s0_exp1") + ) + + def test_optimize_experiments_symmetry_breaking_multiple_markers_warning(self): + # Tests that multiple symmetry markers trigger an ambiguity warning. + doe_obj = DesignOfExperiments( + experiment=[ + RooneyBieglerMultiInputExperimentFlag(hour=2.0, sym_break_flag=2), + RooneyBieglerMultiInputExperimentFlag(hour=4.0, sym_break_flag=2), + ], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + with self.assertLogs("pyomo.contrib.doe.doe", level="WARNING") as cm: + doe_obj.optimize_experiments() + self.assertTrue( + any( + "Multiple variables marked in sym_break_cons" in msg + for msg in cm.output + ) + ) + + def test_lhs_initialization_large_space_emits_warnings(self): + # Tests that very large LHS candidate/combo spaces emit user-facing warnings. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0, y=10.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + with self.assertLogs("pyomo.contrib.doe.doe", level="WARNING") as log_cm: + with warnings.catch_warnings(record=True) as warn_cm: + warnings.simplefilter("always") + with patch( + "pyomo.contrib.doe.doe._combinations", return_value=iter([(0, 1)]) + ): + with patch.object( + doe_obj, + "_compute_fim_at_point_no_prior", + return_value=np.eye(2), + ): + doe_obj.optimize_experiments( + n_exp=2, + init_method="lhs", + init_n_samples=10001, + init_seed=11, + ) + + self.assertTrue( + any("candidate experiment designs" in str(w.message) for w in warn_cm) + ) + self.assertTrue(any("combinations to evaluate" in msg for msg in log_cm.output)) + + def test_lhs_combo_parallel_requested_but_not_used_warns(self): + # Tests that combo-parallel requests warn when thresholds force serial scoring. + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0, y=10.0)], + objective_option="pseudo_trace", + step=1e-2, + solver=self._make_solver(), + ) + with patch.object( + doe_obj, "_compute_fim_at_point_no_prior", return_value=np.eye(2) + ): + with self.assertLogs("pyomo.contrib.doe.doe", level="WARNING") as cm: + doe_obj.optimize_experiments( + n_exp=2, + init_method="lhs", + init_n_samples=2, + init_seed=11, + init_combo_parallel=True, + init_n_workers=2, + init_combo_parallel_threshold=10_000, + ) + + self.assertTrue( + any( + "lhs_combo_parallel=True" in msg and "running serially" in msg + for msg in cm.output + ) + ) + + def test_lhs_missing_bounds_error_message(self): + # Tests that LHS initialization fails fast when experiment inputs lack bounds. + doe_obj = DesignOfExperiments( + experiment=[ + RooneyBieglerMultiExperiment(hour=2.0, hour_bounds=(None, 10.0)) + ], + objective_option="pseudo_trace", + ) + with self.assertRaisesRegex( + ValueError, + r"LHS initialization requires explicit lower and upper bounds on all experiment input variables", + ): + doe_obj.optimize_experiments(init_method="lhs", init_n_samples=2) + if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/doe/tests/test_doe_solve.py b/pyomo/contrib/doe/tests/test_doe_solve.py index 9e31accf894..4b9b04bc3c7 100644 --- a/pyomo/contrib/doe/tests/test_doe_solve.py +++ b/pyomo/contrib/doe/tests/test_doe_solve.py @@ -6,10 +6,14 @@ # Solutions of Sandia, LLC, the U.S. Government retains certain rights in this # software. This software is distributed under the 3-clause BSD License. # ____________________________________________________________________________________ -import json import logging import os, os.path +import subprocess +import tempfile +import time +from itertools import combinations, product from glob import glob +from unittest.mock import patch from pyomo.common.dependencies import ( numpy as np, @@ -25,25 +29,28 @@ if matplotlib_available: matplotlib.use("Agg") -from pyomo.common.fileutils import this_file_dir import pyomo.common.unittest as unittest if not (numpy_available and scipy_available): raise unittest.SkipTest("Pyomo.DoE needs scipy and numpy to run tests") -if scipy_available: - from pyomo.contrib.doe import DesignOfExperiments - from pyomo.contrib.doe.examples.reactor_experiment import ReactorExperiment - from pyomo.contrib.doe.examples.reactor_example import ( - ReactorExperiment as FullReactorExperiment, - run_reactor_doe, - ) - from pyomo.contrib.doe.tests.experiment_class_example_flags import ( - RooneyBieglerExperimentBad, - ) - from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( - RooneyBieglerExperiment, - ) +from pyomo.contrib.doe import DesignOfExperiments +from pyomo.contrib.doe.examples.polynomial import ( + PolynomialExperiment, + run_polynomial_doe, +) +from pyomo.contrib.doe.examples.reactor_experiment import ReactorExperiment +from pyomo.contrib.doe.examples.reactor_example import ( + ReactorExperiment as FullReactorExperiment, + run_reactor_doe, +) +from pyomo.contrib.doe.tests.experiment_class_example_flags import ( + RooneyBieglerExperimentBad, + RooneyBieglerMultiExperiment, +) +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) from pyomo.contrib.doe.utils import rescale_FIM from pyomo.contrib.doe.examples.rooney_biegler_doe_example import run_rooney_biegler_doe @@ -54,12 +61,36 @@ ipopt_available = SolverFactory("ipopt").available() k_aug_available = SolverFactory("k_aug", solver_io="nl", validate=False) -currdir = this_file_dir() -file_path = os.path.join(currdir, "..", "examples", "result.json") -with open(file_path) as f: - data_ex = json.load(f) -data_ex["control_points"] = {float(k): v for k, v in data_ex["control_points"].items()} +def k_aug_runtime_available(): + """ + Check that k_aug is not only discoverable but also runnable in this + environment (e.g., no missing dynamic libraries). + """ + if not k_aug_available.available(False): + return False + exe = k_aug_available.executable() + if not exe: + return False + + try: + # Trigger dynamic loader checks; return code may be nonzero for usage, + # so we inspect output for runtime-linker failures. + proc = subprocess.run( + [exe, "--help"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + except OSError: + return False + + output = (proc.stdout or "") + (proc.stderr or "") + bad_runtime_markers = ("Library not loaded", "dyld:", "libgfortran") + if any(marker in output for marker in bad_runtime_markers): + return False + return True def get_rooney_biegler_data(): @@ -84,6 +115,11 @@ def get_rooney_biegler_experiment(): ) +def _optimize_experiments_param_scenario(results, index=0): + """Return one parameter-scenario entry from optimize_experiments() results.""" + return results["solution"]["param_scenarios"][index] + + def get_FIM_Q_L(doe_obj=None): """ Helper function to retrieve results to compare. @@ -113,10 +149,10 @@ def get_FIM_Q_L(doe_obj=None): for j in model.parameter_names ] sigma_inv = [ - 1 / v**2 for k, v in model.scenario_blocks[0].measurement_error.items() + 1 / v**2 for k, v in model.fd_scenario_blocks[0].measurement_error.items() ] param_vals = np.array( - [[v for k, v in model.scenario_blocks[0].unknown_parameters.items()]] + [[v for k, v in model.fd_scenario_blocks[0].unknown_parameters.items()]] ) FIM_vals_np = np.array(FIM_vals).reshape((n_param, n_param)) @@ -139,7 +175,7 @@ def get_FIM_Q_L(doe_obj=None): def get_standard_args(experiment, fd_method, obj_used): args = {} - args['experiment'] = experiment + args['experiment'] = None if experiment is None else [experiment] args['fd_formula'] = fd_method args['step'] = 1e-3 args['objective_option'] = obj_used @@ -162,6 +198,39 @@ def get_standard_args(experiment, fd_method, obj_used): return args +def get_polynomial_experiment(measurement_error=1.0): + """Build a fresh polynomial experiment with a configurable measurement error.""" + experiment = PolynomialExperiment() + model = experiment.get_labeled_model() + model.measurement_error[model.y] = measurement_error + return experiment + + +def get_polynomial_args( + gradient_method=None, + measurement_error=1.0, + prior_FIM=None, + objective_option="determinant", +): + """Return standard DoE arguments for the public polynomial example.""" + experiment = get_polynomial_experiment(measurement_error=measurement_error) + DoE_args = get_standard_args(experiment, "central", objective_option) + DoE_args["scale_nominal_param_value"] = False + DoE_args["prior_FIM"] = prior_FIM + if gradient_method is not None: + DoE_args["gradient_method"] = gradient_method + return DoE_args + + +def get_expected_polynomial_fim(x1=2.0, x2=3.0, measurement_error=1.0, prior_FIM=None): + """Return the hand-derived polynomial Fisher information matrix.""" + sensitivity = np.array([x1, x2, x1 * x2, 1.0], dtype=float) + fim = np.outer(sensitivity, sensitivity) / (measurement_error**2) + if prior_FIM is not None: + fim = fim + prior_FIM + return fim + + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") @unittest.skipIf(not numpy_available, "Numpy is not available") @unittest.skipIf(not scipy_available, "scipy is not available") @@ -312,7 +381,12 @@ def test_rooney_biegler_obj_cholesky_solve(self): # Note: When using prior_FIM, the relationship FIM = Q.T @ sigma_inv @ Q + prior_FIM self.assertTrue(np.all(np.isclose(FIM, Q.T @ sigma_inv @ Q + prior_FIM))) - def DISABLE_test_reactor_obj_cholesky_solve_bad_prior(self): + # This legacy Cholesky/bad-prior case is kept disabled in + # this PR. It is not part of the active regression signal, and this cleanup is + # focused on replacing active general-purpose coverage with + # Rooney-Biegler or polynomial examples rather than rewriting inactive, + # branch-specific tests. + def DISABLE_test_rooney_biegler_obj_cholesky_solve_bad_prior(self): # [10/2025] This test has been disabled because it frequently # (and randomly) returns "infeasible" when run on Windows. from pyomo.contrib.doe.doe import _SMALL_TOLERANCE_DEFINITENESS @@ -320,7 +394,7 @@ def DISABLE_test_reactor_obj_cholesky_solve_bad_prior(self): fd_method = "central" obj_used = "determinant" - experiment = FullReactorExperiment(data_ex, 10, 3) + experiment = get_rooney_biegler_experiment() DoE_args = get_standard_args(experiment, fd_method, obj_used) @@ -383,12 +457,58 @@ def test_compute_FIM_seq_forward(self): doe_obj.compute_FIM(method="sequential") + def test_compute_FIM_multi_experiment_is_sum_of_experiment_fims(self): + fd_method = "central" + obj_used = "pseudo_trace" + + fim_exp1 = np.array([[12.0, 3.0], [3.0, 8.0]]) + fim_exp2 = np.array([[5.0, 2.0], [2.0, 4.0]]) + prior_fim = np.array([[1.5, 0.1], [0.1, 0.5]]) + + multi_args = get_standard_args( + RooneyBieglerMultiExperiment(hour=1.5, y=9.0), fd_method, obj_used + ) + multi_args["experiment"] = [ + RooneyBieglerMultiExperiment(hour=1.5, y=9.0), + RooneyBieglerMultiExperiment(hour=3.5, y=12.0), + ] + multi_args["prior_FIM"] = prior_fim + doe_multi = DesignOfExperiments(**multi_args) + + def _fake_sequential_fim(*args, **kwargs): + # Return deterministic per-experiment FIMs keyed by the fixed + # experiment input so the aggregation can be asserted exactly. + model = kwargs.get("model") + hour = float(pyo.value(model.hour)) + if np.isclose(hour, 1.5): + doe_multi.seq_FIM = fim_exp1.copy() + elif np.isclose(hour, 3.5): + doe_multi.seq_FIM = fim_exp2.copy() + else: + raise RuntimeError(f"Unexpected hour value in mocked test: {hour}") + + with patch.object( + doe_multi, "_sequential_FIM", side_effect=_fake_sequential_fim + ): + fim_total = doe_multi.compute_FIM(method="sequential") + + fim_expected = fim_exp1 + fim_exp2 + prior_fim + self.assertTrue(np.allclose(fim_total, fim_expected, atol=1e-12)) + self.assertEqual(len(doe_multi._computed_FIM_by_experiment), 2) + self.assertTrue( + np.allclose(doe_multi._computed_FIM_by_experiment[0], fim_exp1, atol=1e-12) + ) + self.assertTrue( + np.allclose(doe_multi._computed_FIM_by_experiment[1], fim_exp2, atol=1e-12) + ) + # This test ensure that compute FIM runs without error using the # `kaug` option. kaug computes the FIM directly so no finite difference # scheme is needed. @unittest.skipIf(not scipy_available, "Scipy is not available") @unittest.skipIf( - not k_aug_available.available(False), "The 'k_aug' command is not available" + not k_aug_runtime_available(), + "The 'k_aug' command is not available or not runnable in this environment", ) def test_compute_FIM_kaug(self): fd_method = "forward" @@ -408,6 +528,24 @@ def test_compute_FIM_kaug(self): np.all(np.isclose(doe_obj.compute_FIM(method="kaug"), expected_FIM)) ) + @unittest.skipIf(not pandas_available, "pandas is not available") + def test_compute_FIM_pynumero(self): + fd_method = "central" + obj_used = "zero" + + experiment = get_rooney_biegler_experiment() + + DoE_args = get_standard_args(experiment, fd_method, obj_used) + DoE_args["gradient_method"] = "pynumero" + + doe_obj = DesignOfExperiments(**DoE_args) + + expected_FIM = np.array( + [[18957.7788694, 4238.27606876], [4238.27606876, 947.52577076]] + ) + + self.assertTrue(np.all(np.isclose(doe_obj.compute_FIM(), expected_FIM))) + # This test ensure that compute FIM runs without error using the # `sequential` option with backward finite differences @unittest.skipIf(not pandas_available, "pandas is not available") @@ -425,43 +563,191 @@ def test_compute_FIM_seq_backward(self): doe_obj.compute_FIM(method="sequential") @unittest.skipIf(not pandas_available, "pandas is not available") - def test_reactor_grid_search(self): + def test_polynomial_grid_search(self): fd_method = "central" obj_used = "determinant" - experiment = FullReactorExperiment(data_ex, 10, 3) + experiment = PolynomialExperiment() DoE_args = get_standard_args(experiment, fd_method, obj_used) doe_obj = DesignOfExperiments(**DoE_args) - # Reduce grid from 3x3 to 2x2 for performance - design_ranges = {"CA[0]": [1, 5, 2], "T[0]": [300, 700, 2]} + # Use a small 2x2 polynomial design grid for a lightweight factorial check. + design_ranges = {"x1": [0, 5, 2], "x2": [0, 5, 2]} doe_obj.compute_FIM_full_factorial( design_ranges=design_ranges, method="sequential" ) - # Check to make sure the lengths of the inputs - # in results object are indeed correct - CA_vals = doe_obj.fim_factorial_results["CA[0]"] - T_vals = doe_obj.fim_factorial_results["T[0]"] + # Check that the factorial results contain the expected 2x2 grid entries. + x1_vals = doe_obj.fim_factorial_results["x1"] + x2_vals = doe_obj.fim_factorial_results["x2"] # assert length is correct (2x2 = 4 evaluations) - self.assertTrue((len(CA_vals) == 4) and (len(T_vals) == 4)) - self.assertTrue((len(set(CA_vals)) == 2) and (len(set(T_vals)) == 2)) + self.assertTrue((len(x1_vals) == 4) and (len(x2_vals) == 4)) + self.assertTrue((len(set(x1_vals)) == 2) and (len(set(x2_vals)) == 2)) - # assert unique values are correct + # Check that each polynomial design variable spans the requested grid values. self.assertTrue( - (set(CA_vals).issuperset(set([1, 5]))) - and (set(T_vals).issuperset(set([300, 700]))) + (set(x1_vals).issuperset(set([0, 5]))) + and (set(x2_vals).issuperset(set([0, 5]))) + ) + + def test_rooney_biegler_run_doe_pynumero(self): + fd_method = "central" + obj_used = "determinant" + + experiment = get_rooney_biegler_experiment() + + DoE_args = get_standard_args(experiment, fd_method, obj_used) + DoE_args["gradient_method"] = "pynumero" + + doe_obj = DesignOfExperiments(**DoE_args) + # Rooney-Biegler determinant solves are numerically more stable with a + # prior. Use the FIM at the current nominal design as the prior so this + # symbolic run_doe() test exercises the intended backend without + # relying on a singular starting information matrix. + prior_FIM = doe_obj.compute_FIM() + doe_obj.prior_FIM = prior_FIM + doe_obj.run_doe() + + self.assertEqual(doe_obj.results["Solver Status"], "ok") + + FIM, Q, L, sigma_inv = get_FIM_Q_L(doe_obj=doe_obj) + self.assertTrue(np.all(np.isclose(FIM, L @ L.T))) + self.assertTrue(np.all(np.isclose(FIM, Q.T @ sigma_inv @ Q + prior_FIM))) + + def test_rooney_biegler_run_doe_determinant_regression(self): + """Check a stable Rooney-Biegler optimum fingerprint against expected values.""" + experiment = get_rooney_biegler_experiment() + DoE_args = get_standard_args(experiment, "central", "determinant") + DoE_args["gradient_method"] = "pynumero" + doe_obj = DesignOfExperiments(**DoE_args) + # Rooney-Biegler determinant solves are numerically more stable with a + # prior. Use the FIM at the current nominal design as the prior so this + # symbolic regression test keeps the determinant problem well + # conditioned while preserving the same backend configuration. + prior_FIM = doe_obj.compute_FIM() + doe_obj.prior_FIM = prior_FIM + doe_obj.run_doe() + + self.assertEqual(doe_obj.results["Solver Status"], "ok") + self.assertEqual( + str(doe_obj.results["Termination Condition"]).lower(), "optimal" + ) + + design = doe_obj.results["Experiment Design"] + self.assertAlmostEqual(design[0], 9.999999776254937, places=4) + fim = np.array(doe_obj.results["FIM"]) + self.assertAlmostEqual(fim[0, 0], 41155.59271917, places=2) + self.assertAlmostEqual(fim[1, 1], 973.06126181, places=2) + self.assertAlmostEqual( + doe_obj.results["log10 D-opt"], 7.179982499524086, places=4 + ) + self.assertAlmostEqual( + doe_obj.results["log10 A-opt"], -2.5554049159721415, places=4 ) + def test_rooney_biegler_run_doe_pynumero_objective_matrix(self): + """Exercise the symbolic Rooney-Biegler run_doe path across objective options.""" + test_cases = ["trace", "determinant", "zero"] + + for objective_option in test_cases: + with self.subTest(objective_option=objective_option): + experiment = get_rooney_biegler_experiment() + DoE_args = get_standard_args(experiment, "central", objective_option) + DoE_args["gradient_method"] = "pynumero" + + doe_obj = DesignOfExperiments(**DoE_args) + # Rooney-Biegler run_doe() cases are more stable with the + # nominal-design FIM supplied as a prior, so each objective is + # exercised under the symbolic backend without starting from a + # nearly singular information matrix. + prior_FIM = doe_obj.compute_FIM() + doe_obj.prior_FIM = prior_FIM + doe_obj.run_doe() + + self.assertEqual(doe_obj.results["Solver Status"], "ok") + + FIM, Q, L, sigma_inv = get_FIM_Q_L(doe_obj=doe_obj) + if objective_option == "determinant": + self.assertTrue(np.all(np.isclose(FIM, L @ L.T))) + self.assertTrue( + np.all(np.isclose(FIM, Q.T @ sigma_inv @ Q + prior_FIM)) + ) + + def test_polynomial_example_compute_fim_pynumero(self): + """Check that the transplanted polynomial example computes the expected FIM.""" + fim = run_polynomial_doe() + expected = get_expected_polynomial_fim() + self.assertEqual(fim.shape, expected.shape) + self.assertTrue(np.allclose(fim, expected)) + + def test_polynomial_example_measurement_error_scaling(self): + """Check that doubling the measurement error scales the FIM by one quarter.""" + fim_sigma_one = DesignOfExperiments( + **get_polynomial_args(gradient_method="pynumero", measurement_error=1.0) + ).compute_FIM() + fim_sigma_two = DesignOfExperiments( + **get_polynomial_args(gradient_method="pynumero", measurement_error=2.0) + ).compute_FIM() + + self.assertTrue(np.allclose(fim_sigma_two, fim_sigma_one / 4.0)) + self.assertTrue( + np.allclose( + fim_sigma_two, get_expected_polynomial_fim(measurement_error=2.0) + ) + ) + + def test_polynomial_example_prior_fim_adds_directly(self): + """Check that the polynomial example adds prior information entry-wise.""" + prior_FIM = np.diag([1.0, 2.0, 3.0, 4.0]) + doe_obj = DesignOfExperiments( + **get_polynomial_args( + gradient_method="pynumero", measurement_error=1.0, prior_FIM=prior_FIM + ) + ) + expected = get_expected_polynomial_fim(prior_FIM=prior_FIM) + + self.assertTrue(np.allclose(doe_obj.compute_FIM(), expected)) + + def test_polynomial_example_run_doe_smoke(self): + """Check that the public polynomial example can solve a tiny DoE problem. + Also do a regression test to check that the solution returned stays correct over time + """ + prior_FIM = np.eye(4) + doe_obj = DesignOfExperiments( + **get_polynomial_args( + gradient_method="pynumero", + prior_FIM=prior_FIM, + objective_option="determinant", + ) + ) + + doe_obj.run_doe() + + self.assertEqual(doe_obj.results["Solver Status"], "ok") + self.assertEqual( + str(doe_obj.results["Termination Condition"]).lower(), "optimal" + ) + design = doe_obj.results["Experiment Design"] + self.assertAlmostEqual(design[0], 5.0, places=4) + self.assertAlmostEqual(design[1], 5.0, places=4) + + self.assertAlmostEqual( + doe_obj.results["log10 D-opt"], 2.830588683545922, places=4 + ) + + fim = np.array(doe_obj.results["FIM"]) + self.assertAlmostEqual(fim[0, 0], 26.00000045, places=4) + self.assertAlmostEqual(fim[3, 3], 2.0, places=4) + def test_rescale_FIM(self): fd_method = "central" obj_used = "determinant" - experiment = FullReactorExperiment(data_ex, 10, 3) + experiment = get_rooney_biegler_experiment() # With parameter scaling DoE_args = get_standard_args(experiment, fd_method, obj_used) @@ -473,22 +759,18 @@ def test_rescale_FIM(self): DoE_args2["scale_nominal_param_value"] = False doe_obj2 = DesignOfExperiments(**DoE_args2) - # Run both problems - doe_obj.run_doe() - doe_obj2.run_doe() - - # Extract FIM values - FIM, Q, L, sigma_inv = get_FIM_Q_L(doe_obj=doe_obj) - FIM2, Q2, L2, sigma_inv2 = get_FIM_Q_L(doe_obj=doe_obj2) + # Compare scaled and unscaled FIMs at the same nominal design. For the + # Rooney-Biegler replacement, this avoids introducing determinant-solve + # instability that is unrelated to the rescaling utility itself. + FIM = doe_obj.compute_FIM() + FIM2 = doe_obj2.compute_FIM() # Get rescaled FIM from the scaled version param_vals = np.array( [ [ v - for k, v in doe_obj.model.scenario_blocks[ - 0 - ].unknown_parameters.items() + for k, v in doe_obj.compute_FIM_model.unknown_parameters.items() ] ] ) @@ -499,11 +781,11 @@ def test_rescale_FIM(self): self.assertTrue(np.all(np.isclose(FIM2, resc_FIM))) @unittest.skipIf(not pandas_available, "pandas is not available") - def test_reactor_solve_bad_model(self): + def test_rooney_biegler_solve_bad_model(self): fd_method = "central" obj_used = "determinant" - # Use RooneyBiegler bad example (faster than reactor bad example) + # Use the Rooney-Biegler bad example as the lightweight bad-model case. experiment = RooneyBieglerExperimentBad( data=get_rooney_biegler_data(), theta={'asymptote': 15, 'rate_constant': 0.5}, @@ -522,11 +804,11 @@ def test_reactor_solve_bad_model(self): doe_obj.run_doe() @unittest.skipIf(not pandas_available, "pandas is not available") - def test_reactor_grid_search_bad_model(self): + def test_rooney_biegler_grid_search_bad_model(self): fd_method = "central" obj_used = "determinant" - # Use RooneyBiegler bad example (faster than reactor bad example) + # Use the Rooney-Biegler bad example as the lightweight bad-model case. experiment = RooneyBieglerExperimentBad( data=get_rooney_biegler_data(), theta={'asymptote': 15, 'rate_constant': 0.5}, @@ -559,72 +841,73 @@ def test_reactor_grid_search_bad_model(self): @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") @unittest.skipIf(not numpy_available, "Numpy is not available") class TestDoe(unittest.TestCase): - def test_doe_full_factorial(self): + def test_polynomial_full_factorial(self): + """Check 2D factorial FIM metrics on the lightweight polynomial example.""" log10_D_opt_expected = [ - 11.77343778527225, - 13.137792359064383, - 13.182167857699808, - 14.54652243150573, + 3.771625936657566, + 5.566143287412265, + 5.910363131426261, + 6.173537519214883, ] log10_A_opt_expected = [ - 5.59357268009304, - 5.613318615148643, - 5.945755198204368, - 5.965501133259909, + 3.771846315265457, + 5.5661468254334245, + 5.910364732979566, + 6.173538392929174, ] log10_E_opt_expected = [ - 0.27981268741620413, - 1.3086595026369012, - 0.6319952055040333, - 1.6608420207466377, + -4.821637332766436e-17, + -7.8206957537542e-13, + -2.4960652144337014e-12, + -1.0111706376862954e-10, ] log10_ME_opt_expected = [ - 5.221185311075697, - 4.244741560076784, - 5.221185311062606, - 4.244741560083524, + 3.771625936657538, + 5.566143287414164, + 5.910363131438886, + 6.173537519245602, ] eigval_min_expected = [ - 1.9046390638130666, - 20.354456134677426, - 4.285437893696232, - 45.797526302234304, + 0.9999999999999999, + 0.9999999999981992, + 0.9999999999942526, + 0.9999999997671694, ] eigval_max_expected = [ - 316955.2855492114, - 357602.92523637977, - 713149.3924857995, - 804606.58178139165, + 5910.523340060432, + 368250.4510100104, + 813510.4413073618, + 1491205.5769174611, ] det_FIM_expected = [ - 593523317093.4525, - 13733851875566.766, - 15211353450350.424, - 351983602166961.56, + 5910.523340060814, + 368250.45101058815, + 813510.4413089894, + 1491205.5769048228, ] trace_FIM_expected = [ - 392258.78617108597, - 410505.1549241871, - 882582.2688850109, - 923636.598578955, + 5913.523340060433, + 368253.4510100104, + 813513.4413073619, + 1491208.5769174611, ] - ff = run_reactor_doe( - n_points_for_C0=2, - n_points_for_T0=2, - compute_FIM_full_factorial=False, - plot_factorial_results=False, - run_optimal_doe=False, - ) - ff.compute_FIM_full_factorial( - design_ranges={"CA[0]": [1, 1.5, 2], "T[0]": [350, 400, 2]} - ) + experiment = PolynomialExperiment() + DoE_args = get_standard_args(experiment, "central", "trace") + DoE_args["scale_nominal_param_value"] = False + # The polynomial model has one output and four parameters, so the raw + # outer-product FIM is rank one. Seed the factorial sweep with an + # identity prior to keep the metric regressions positive definite. + DoE_args["prior_FIM"] = np.eye(4) + + ff = DesignOfExperiments(**DoE_args) + ff.compute_FIM_full_factorial(design_ranges={"x1": [0, 5, 2], "x2": [0, 5, 2]}) ff_results = ff.fim_factorial_results @@ -673,120 +956,207 @@ def test_rooney_biegler_doe_example(self): Tests the Design of Experiments (DoE) functionality, including plotting logic when matplotlib is available, without displaying GUI windows. """ - file_prefix = "rooney_biegler" - - # Cleanup function for generated files - def cleanup_file(): - generated_files = glob(f"{file_prefix}_*.png") - for f in generated_files: - try: - os.remove(f) - except OSError: - pass - - self.addCleanup(cleanup_file) - - # Run with draw_factorial_figure conditional on matplotlib availability - # Test D-optimality - results_D = run_rooney_biegler_doe( - optimization_objective="determinant", - compute_FIM_full_factorial=True, - draw_factorial_figure=matplotlib_available, - design_range={'hour': [0, 10, 3]}, - tee=False, - ) - - # Test A-optimality - results_A = run_rooney_biegler_doe( - optimization_objective="trace", - compute_FIM_full_factorial=False, - draw_factorial_figure=False, - design_range={'hour': [0, 10, 3]}, - tee=False, - ) - - # Assertions for Numerical Results - self.assertEqual("determinant", results_D["optimization"]["objective_type"]) - self.assertEqual("trace", results_A["optimization"]["objective_type"]) - - # Test D-optimality optimization results - D_opt_value_expected = 6.864794717802814 - D_opt_design_value_expected = 10.0 # approximately 9.999999472662282 - - D_opt_value = results_D["optimization"]["value"] - D_opt_design_value = results_D["optimization"]["design"][0] - - self.assertAlmostEqual(D_opt_value, D_opt_value_expected, places=4) - self.assertAlmostEqual( - D_opt_design_value, D_opt_design_value_expected, places=4 - ) + with tempfile.TemporaryDirectory() as tmpdir: + file_prefix = os.path.join(tmpdir, "rooney_biegler") + prev_cwd = os.getcwd() + os.chdir(tmpdir) + self.addCleanup(os.chdir, prev_cwd) + + # Run with draw_factorial_figure conditional on matplotlib availability + # Test D-optimality + results_D = run_rooney_biegler_doe( + optimization_objective="determinant", + compute_FIM_full_factorial=True, + draw_factorial_figure=matplotlib_available, + design_range={'hour': [0, 10, 3]}, + tee=False, + ) - # Test A-optimality optimization results - A_opt_value_expected = -2.236424205953928 - A_opt_design_value_expected = 10.0 # approximately 9.999955457176451 + # Test A-optimality + results_A = run_rooney_biegler_doe( + optimization_objective="trace", + compute_FIM_full_factorial=False, + draw_factorial_figure=False, + design_range={'hour': [0, 10, 3]}, + tee=False, + ) - A_opt_value = results_A["optimization"]["value"] - A_opt_design_value = results_A["optimization"]["design"][0] + # Assertions for Numerical Results + self.assertEqual("determinant", results_D["optimization"]["objective_type"]) + self.assertEqual("trace", results_A["optimization"]["objective_type"]) - self.assertAlmostEqual(A_opt_value, A_opt_value_expected, places=4) - self.assertAlmostEqual( - A_opt_design_value, A_opt_design_value_expected, places=4 - ) - - # Assertions for Full Factorial Results - self.assertIn("results_dict", results_D) - results_dict = results_D["results_dict"] - self.assertIsInstance(results_dict, dict) - self.assertGreater(len(results_dict), 0, "results_dict should not be empty") - - # Expected values for design_range={'hour': [0, 10, 3]} - # These are the 3 data points from the full factorial grid - expected_log10_D_opt = [6.583798747893548, 6.691228337572129, 6.864794726228617] - expected_log10_A_opt = [ - -1.9574859220185146, - -2.0268526846104975, - -2.236424954559946, - ] - expected_log10_pseudo_A_opt = [ - 4.62631282587503, - 4.6643756529616285, - 4.628369771668666, - ] - expected_log10_E_opt = [ - 1.9584199467177335, - 2.0278567624056967, - 2.2381970919918426, - ] - expected_log10_ME_opt = [ - 2.666958854458125, - 2.6355148127607713, - 2.3884005422449364, - ] + # Test D-optimality optimization results + D_opt_value_expected = 6.864794717802814 + D_opt_design_value_expected = 10.0 # approximately 9.999999472662282 - # Verify structure and values using assertStructuredAlmostEqual - self.assertStructuredAlmostEqual( - results_dict["log10 D-opt"], expected_log10_D_opt, abstol=1e-4 + D_opt_value = results_D["optimization"]["value"] + D_opt_design_value = results_D["optimization"]["design"][0] + + self.assertAlmostEqual(D_opt_value, D_opt_value_expected, places=4) + self.assertAlmostEqual( + D_opt_design_value, D_opt_design_value_expected, places=4 + ) + + # Test A-optimality optimization results + A_opt_value_expected = -2.236424205953928 + A_opt_design_value_expected = 10.0 # approximately 9.999955457176451 + + A_opt_value = results_A["optimization"]["value"] + A_opt_design_value = results_A["optimization"]["design"][0] + + self.assertAlmostEqual(A_opt_value, A_opt_value_expected, places=4) + self.assertAlmostEqual( + A_opt_design_value, A_opt_design_value_expected, places=4 + ) + + # Assertions for Full Factorial Results + self.assertIn("results_dict", results_D) + results_dict = results_D["results_dict"] + self.assertIsInstance(results_dict, dict) + self.assertGreater(len(results_dict), 0, "results_dict should not be empty") + + # Expected values for design_range={'hour': [0, 10, 3]} + # These are the 3 data points from the full factorial grid + expected_log10_D_opt = [ + 6.583798747893548, + 6.691228337572129, + 6.864794726228617, + ] + expected_log10_A_opt = [ + -1.9574859220185146, + -2.0268526846104975, + -2.236424954559946, + ] + expected_log10_pseudo_A_opt = [ + 4.62631282587503, + 4.6643756529616285, + 4.628369771668666, + ] + expected_log10_E_opt = [ + 1.9584199467177335, + 2.0278567624056967, + 2.2381970919918426, + ] + expected_log10_ME_opt = [ + 2.666958854458125, + 2.6355148127607713, + 2.3884005422449364, + ] + + # Verify structure and values using assertStructuredAlmostEqual + self.assertStructuredAlmostEqual( + results_dict["log10 D-opt"], expected_log10_D_opt, abstol=1e-4 + ) + self.assertStructuredAlmostEqual( + results_dict["log10 A-opt"], expected_log10_A_opt, abstol=1e-4 + ) + self.assertStructuredAlmostEqual( + results_dict["log10 pseudo A-opt"], + expected_log10_pseudo_A_opt, + abstol=1e-4, + ) + self.assertStructuredAlmostEqual( + results_dict["log10 E-opt"], expected_log10_E_opt, abstol=1e-4 + ) + self.assertStructuredAlmostEqual( + results_dict["log10 ME-opt"], expected_log10_ME_opt, abstol=1e-4 + ) + + # Plot-related assertions only when matplotlib is available + if matplotlib_available: + # Check that draw_factorial_figure actually created the file + expected_d_plot = f"{file_prefix}_D_opt.png" + self.assertTrue( + os.path.exists(expected_d_plot), + f"Expected plot file '{expected_d_plot}' was not created.", + ) + + @unittest.skipUnless(pandas_available, "test requires pandas") + @unittest.skipUnless(ipopt_available, "test requires ipopt") + def test_rooney_biegler_factorial_results_dataframe_schema(self): + """Check the schema of the Rooney-Biegler factorial-results table.""" + experiment = run_rooney_biegler_doe()["experiment"] + DoE_args = get_standard_args(experiment, "central", "trace") + DoE_args["gradient_method"] = "pynumero" + doe_obj = DesignOfExperiments(**DoE_args) + + results = doe_obj.compute_FIM_full_factorial(design_ranges={"hour": [0, 10, 3]}) + results_pd = pd.DataFrame(results) + + expected_columns = { + "hour", + "log10 D-opt", + "log10 A-opt", + "log10 pseudo A-opt", + "log10 E-opt", + "log10 ME-opt", + "eigval_min", + "eigval_max", + "det_FIM", + "trace_cov", + "trace_FIM", + "solve_time", + } + + self.assertTrue(expected_columns.issubset(results_pd.columns)) + self.assertEqual(len(results_pd), 3) + self.assertEqual(sorted(results_pd["hour"].tolist()), [0.0, 5.0, 10.0]) + self.assertTrue(np.all(np.isfinite(results_pd["solve_time"]))) + + @unittest.skipUnless(pandas_available, "test requires pandas") + @unittest.skipUnless(ipopt_available, "test requires ipopt") + def test_draw_factorial_figure_accepts_dataframe_input(self): + """Check draw_factorial_figure accepts a DataFrame and stores filtered rows.""" + doe_obj = DesignOfExperiments( + **get_polynomial_args( + gradient_method="pynumero", objective_option="determinant" + ) ) - self.assertStructuredAlmostEqual( - results_dict["log10 A-opt"], expected_log10_A_opt, abstol=1e-4 + + results = doe_obj.compute_FIM_full_factorial( + design_ranges={"x1": [0, 5, 2], "x2": [0, 5, 2]} ) - self.assertStructuredAlmostEqual( - results_dict["log10 pseudo A-opt"], expected_log10_pseudo_A_opt, abstol=1e-4 + results_pd = pd.DataFrame(results) + + doe_obj.draw_factorial_figure( + results=results_pd, + sensitivity_design_variables=["x1"], + fixed_design_variables={"x2": 0.0}, + full_design_variable_names=["x1", "x2"], + log_scale=False, + figure_file_name=None, ) - self.assertStructuredAlmostEqual( - results_dict["log10 E-opt"], expected_log10_E_opt, abstol=1e-4 + + filtered = doe_obj.figure_result_data + self.assertIsInstance(filtered, pd.DataFrame) + self.assertEqual(len(filtered), 2) + self.assertTrue(np.allclose(filtered["x2"].values, 0.0)) + self.assertEqual(sorted(filtered["x1"].tolist()), [0.0, 5.0]) + + @unittest.skipUnless(pandas_available, "test requires pandas") + @unittest.skipUnless(ipopt_available, "test requires ipopt") + def test_draw_factorial_figure_bad_fixed_variable_raises(self): + """Check draw_factorial_figure rejects unknown fixed design variables.""" + doe_obj = DesignOfExperiments( + **get_polynomial_args( + gradient_method="pynumero", objective_option="determinant" + ) ) - self.assertStructuredAlmostEqual( - results_dict["log10 ME-opt"], expected_log10_ME_opt, abstol=1e-4 + + results = doe_obj.compute_FIM_full_factorial( + design_ranges={"x1": [0, 5, 3], "x2": [0, 5, 3]} ) - # Plot-related assertions only when matplotlib is available - if matplotlib_available: - # Check that draw_factorial_figure actually created the file - expected_d_plot = f"{file_prefix}_D_opt.png" - self.assertTrue( - os.path.exists(expected_d_plot), - f"Expected plot file '{expected_d_plot}' was not created.", + with self.assertRaisesRegex( + ValueError, "Fixed design variables do not all appear" + ): + doe_obj.draw_factorial_figure( + results=results, + sensitivity_design_variables=["x1"], + fixed_design_variables={"bad_name": 5.0}, + full_design_variable_names=["x1", "x2"], + log_scale=False, + figure_file_name=None, ) @@ -803,122 +1173,1424 @@ def test_doe_1D_plotting_function(self): creates a matplotlib figure. We do NOT test visual correctness. """ - # File prefix for saved plots - # Define prefixes for the two runs - prefix_linear = "rooney_linear" - prefix_log = "rooney_log" + with tempfile.TemporaryDirectory() as tmpdir: + # File prefix for saved plots + # Define prefixes for the two runs + prefix_linear = os.path.join(tmpdir, "rooney_linear") + prefix_log = os.path.join(tmpdir, "rooney_log") - # Clean up any existing plot files from test runs - def cleanup_files(): - files_to_remove = glob("rooney_*.png") - for f in files_to_remove: - try: - os.remove(f) - except OSError: - pass - plt.close('all') + self.addCleanup(plt.close, 'all') - self.addCleanup(cleanup_files) + fd_method = "central" + obj_used = "trace" - fd_method = "central" - obj_used = "trace" + experiment = run_rooney_biegler_doe()["experiment"] - experiment = run_rooney_biegler_doe()["experiment"] + DoE_args = get_standard_args(experiment, fd_method, obj_used) + doe_obj = DesignOfExperiments(**DoE_args) - DoE_args = get_standard_args(experiment, fd_method, obj_used) - doe_obj = DesignOfExperiments(**DoE_args) + doe_obj.compute_FIM_full_factorial(design_ranges={'hour': [0, 10, 1]}) - doe_obj.compute_FIM_full_factorial(design_ranges={'hour': [0, 10, 1]}) + # Call the plotting function for linear scale + doe_obj.draw_factorial_figure( + sensitivity_design_variables=['hour'], + fixed_design_variables={}, + log_scale=False, + figure_file_name=prefix_linear, + ) - # Call the plotting function for linear scale - doe_obj.draw_factorial_figure( - sensitivity_design_variables=['hour'], - fixed_design_variables={}, - log_scale=False, - figure_file_name=prefix_linear, + # Call the plotting function for log scale + doe_obj.draw_factorial_figure( + sensitivity_design_variables=['hour'], + fixed_design_variables={}, + log_scale=True, + figure_file_name=prefix_log, + ) + + # Verify that the linear scale plots were also created + # Check that we found exactly 5 files (A, D, E, ME, pseudo_A) + expected_plot_linear = glob(f"{prefix_linear}*.png") + self.assertEqual( + len(expected_plot_linear), + 5, + f"Expected 5 plot files, but found {len(expected_plot_linear)}. Files found: {expected_plot_linear}", + ) + + # Verify that the log scale plots were also created + expected_plot_log = glob(f"{prefix_log}*.png") + self.assertEqual( + len(expected_plot_log), + 5, + f"Expected 5 plot files, but found {len(expected_plot_log)}. Files found: {expected_plot_log}", + ) + + def test_polynomial_2D_plotting_function(self): + # Use the lightweight polynomial example for generic 2D factorial plotting. + plt = matplotlib.pyplot + + with tempfile.TemporaryDirectory() as tmpdir: + # File prefix for saved plots + prefix_linear = os.path.join(tmpdir, "polynomial_linear") + prefix_log = os.path.join(tmpdir, "polynomial_log") + + self.addCleanup(plt.close, 'all') + + experiment = PolynomialExperiment() + DoE_args = get_standard_args(experiment, "central", "determinant") + DoE_args["gradient_method"] = "pynumero" + DoE_args["scale_nominal_param_value"] = False + + doe_obj = DesignOfExperiments(**DoE_args) + # Build polynomial factorial results and draw the linear-scale 2D plots. + doe_obj.compute_FIM_full_factorial( + design_ranges={"x1": [0, 5, 2], "x2": [0, 5, 2]} + ) + doe_obj.draw_factorial_figure( + sensitivity_design_variables=["x1", "x2"], + fixed_design_variables={}, + full_design_variable_names=["x1", "x2"], + figure_file_name=prefix_linear, + log_scale=False, + ) + + # Verify that the linear scale plots were also created + # Check that we found exactly 5 files (A, D, E, ME, pseudo_A) + expected_plot_linear = glob(f"{prefix_linear}*.png") + self.assertTrue( + len(expected_plot_linear) == 5, + f"Expected 5 plot files, but found {len(expected_plot_linear)}. Files found: {expected_plot_linear}", + ) + + # Reuse the same factorial results to draw the log-scale 2D plots. + doe_obj.draw_factorial_figure( + sensitivity_design_variables=["x1", "x2"], + fixed_design_variables={}, + full_design_variable_names=["x1", "x2"], + figure_file_name=prefix_log, + log_scale=True, + ) + + # Verify that the log scale plots were also created + # Check that we found exactly 5 files (A, D, E, ME, pseudo_A) + expected_plot_log = glob(f"{prefix_log}*.png") + self.assertTrue( + len(expected_plot_log) == 5, + f"Expected 5 plot files, but found {len(expected_plot_log)}. Files found: {expected_plot_log}", + ) + + +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") +@unittest.skipIf(not numpy_available, "Numpy is not available") +@unittest.skipIf(not scipy_available, "scipy is not available") +class TestOptimizeExperimentsAlgorithm(unittest.TestCase): + def _make_template_doe(self, objective_option="pseudo_trace"): + exp = RooneyBieglerMultiExperiment(hour=2.0, y=10.0) + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + return DesignOfExperiments( + experiment=[exp], + objective_option=objective_option, + step=1e-2, + solver=solver, ) - # Call the plotting function for log scale - doe_obj.draw_factorial_figure( - sensitivity_design_variables=['hour'], - fixed_design_variables={}, - log_scale=True, - figure_file_name=prefix_log, + def _build_template_model_for_multi_experiment(self, doe_obj, n_exp): + doe_obj.model.param_scenario_blocks = pyo.Block(range(1)) + doe_obj.model.param_scenario_blocks[0].exp_blocks = pyo.Block(range(n_exp)) + for k in range(n_exp): + doe_obj.create_doe_model( + model=doe_obj.model.param_scenario_blocks[0].exp_blocks[k], + experiment_index=0, + _for_multi_experiment=True, + ) + + def test_evaluate_objective_from_fim_numerical_values(self): + fim = np.array([[4.0, 1.0], [1.0, 3.0]]) + expected_det = 11.0 + expected_pseudo_trace = 7.0 + expected_trace = 7.0 / 11.0 + + doe_det = self._make_template_doe("determinant") + self.assertAlmostEqual( + doe_det._evaluate_objective_from_fim(fim), expected_det, places=10 ) - # Verify that the linear scale plots were also created - # Check that we found exactly 5 files (A, D, E, ME, pseudo_A) - expected_plot_linear = glob(f"{prefix_linear}*.png") - self.assertEqual( - len(expected_plot_linear), - 5, - f"Expected 5 plot files, but found {len(expected_plot_linear)}. Files found: {expected_plot_linear}", + doe_ptr = self._make_template_doe("pseudo_trace") + self.assertAlmostEqual( + doe_ptr._evaluate_objective_from_fim(fim), expected_pseudo_trace, places=10 ) - # Verify that the log scale plots were also created - expected_plot_log = glob(f"{prefix_log}*.png") - self.assertEqual( - len(expected_plot_log), - 5, - f"Expected 5 plot files, but found {len(expected_plot_log)}. Files found: {expected_plot_log}", + doe_tr = self._make_template_doe("trace") + self.assertAlmostEqual( + doe_tr._evaluate_objective_from_fim(fim), expected_trace, places=10 ) - def test_doe_2D_plotting_function(self): - # For 2D plotting we will use the Rooney-Biegler example in doe/examples - plt = matplotlib.pyplot + def test_evaluate_objective_from_fim_fallback_paths(self): + singular_fim = np.zeros((2, 2)) - # File prefix for saved plots - prefix_linear = "reactor_linear" - prefix_log = "reactor_log" - - # Clean up any existing plot files from test runs - def cleanup_files(): - files_to_remove = glob("reactor_*.png") - for f in files_to_remove: - try: - os.remove(f) - except OSError: - pass - plt.close('all') - - self.addCleanup(cleanup_files) - - # Run the reactor example - run_reactor_doe( - n_points_for_C0=1, - n_points_for_T0=1, - compute_FIM_full_factorial=True, - plot_factorial_results=True, - figure_file_name=prefix_linear, - log_scale=False, - run_optimal_doe=False, + doe_trace = self._make_template_doe("trace") + self.assertEqual(doe_trace._evaluate_objective_from_fim(singular_fim), np.inf) + + doe_zero = self._make_template_doe("zero") + self.assertEqual(doe_zero._evaluate_objective_from_fim(singular_fim), 0.0) + + def test_compute_fim_at_point_no_prior_restores_prior(self): + doe_no_prior = self._make_template_doe("pseudo_trace") + doe_no_prior.prior_FIM = np.zeros((2, 2)) + expected = doe_no_prior.compute_FIM(method="sequential") + + doe = self._make_template_doe("pseudo_trace") + saved_prior = np.array([[7.0, 0.0], [0.0, 5.0]]) + doe.prior_FIM = saved_prior + got = doe._compute_fim_at_point_no_prior(experiment_index=0, input_values=[2.0]) + + self.assertTrue(np.allclose(got, expected, atol=1e-8)) + self.assertIs(doe.prior_FIM, saved_prior) + self.assertTrue(np.allclose(doe.prior_FIM, saved_prior, atol=1e-12)) + + def test_compute_fim_at_point_no_prior_exception_fallback_zero(self): + doe = self._make_template_doe("pseudo_trace") + saved_prior = np.array([[3.0, 0.0], [0.0, 4.0]]) + doe.prior_FIM = saved_prior + + with patch.object(doe, "_sequential_FIM", side_effect=RuntimeError("boom")): + with self.assertLogs("pyomo.contrib.doe.doe", level="WARNING") as log_cm: + got = doe._compute_fim_at_point_no_prior( + experiment_index=0, input_values=[2.0] + ) + + self.assertTrue(np.allclose(got, np.zeros((2, 2)))) + self.assertIs(doe.prior_FIM, saved_prior) + self.assertTrue( + any("Using zero matrix as fallback" in msg for msg in log_cm.output) + ) + + def test_optimize_experiments_cholesky_jitter_branch(self): + # Force the positive-definiteness check down the jitter path and verify + # the matrix passed into Cholesky includes the expected diagonal shift. + doe = self._make_template_doe("determinant") + + from pyomo.contrib.doe.doe import _SMALL_TOLERANCE_DEFINITENESS + + original_solve = doe.solver.solve + original_eigvals = np.linalg.eigvals + original_cholesky = np.linalg.cholesky + solve_count = {"n": 0} + eig_call_count = {"n": 0} + cholesky_inputs = [] + + class _MockSolverInfo: + status = "ok" + termination_condition = "optimal" + message = "mock-solve" + + class _MockResults: + solver = _MockSolverInfo() + + def _solve_first_real_then_mock(*args, **kwargs): + solve_count["n"] += 1 + if solve_count["n"] == 1: + return original_solve(*args, **kwargs) + return _MockResults() + + def _eigvals_force_jitter(*args, **kwargs): + eig_call_count["n"] += 1 + if eig_call_count["n"] == 1: + return np.array([-1.0, 1.0]) + return original_eigvals(*args, **kwargs) + + def _capture_cholesky(mat, *args, **kwargs): + cholesky_inputs.append(np.array(mat, copy=True)) + return original_cholesky(mat, *args, **kwargs) + + with patch( + "pyomo.contrib.doe.doe.np.linalg.eigvals", side_effect=_eigvals_force_jitter + ) as eigvals_mock: + with patch( + "pyomo.contrib.doe.doe.np.linalg.cholesky", + side_effect=_capture_cholesky, + ): + with patch.object( + doe.solver, "solve", side_effect=_solve_first_real_then_mock + ): + doe.optimize_experiments(n_exp=1) + + scenario = doe.model.param_scenario_blocks[0] + total_fim = np.array( + _optimize_experiments_param_scenario(doe.results)["total_fim"] + ) + expected_cholesky_input = total_fim + _SMALL_TOLERANCE_DEFINITENESS * np.eye( + total_fim.shape[0] + ) + param_names = list(scenario.exp_blocks[0].parameter_names) + L_vals = np.array( + [ + [pyo.value(scenario.obj_cons.L[p, q]) for q in param_names] + for p in param_names + ] + ) + + self.assertEqual(doe.results["optimization_solve"]["status"], "ok") + self.assertGreaterEqual(eigvals_mock.call_count, 1) + self.assertTrue(cholesky_inputs) + self.assertTrue( + any( + np.allclose(chol_arg, expected_cholesky_input, atol=1e-12) + for chol_arg in cholesky_inputs + ) + ) + self.assertTrue( + np.allclose(L_vals @ L_vals.T, expected_cholesky_input, atol=1e-8) + ) + + def test_lhs_initialize_experiments_matches_bruteforce_combo(self): + # Compare the helper's chosen initial design directly against an + # independent brute-force scorer so this test isolates LHS selection + # instead of the later NLP solve and result-payload bookkeeping. + n_exp = 2 + lhs_n_samples = 3 + lhs_seed = 17 + + # Build expected best combo using the same helper path and objective scoring. + expected_obj = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(expected_obj, n_exp=n_exp) + + first_exp_block = expected_obj.model.param_scenario_blocks[0].exp_blocks[0] + exp_input_vars = expected_obj._get_experiment_input_vars(first_exp_block) + lb_vals = np.array([v.lb for v in exp_input_vars]) + ub_vals = np.array([v.ub for v in exp_input_vars]) + + rng = np.random.default_rng(lhs_seed) + from scipy.stats.qmc import LatinHypercube + + per_dim_samples = [] + for i in range(len(exp_input_vars)): + dim_seed = int(rng.integers(0, 2**31)) + sampler = LatinHypercube(d=1, seed=dim_seed) + s_unit = sampler.random(n=lhs_n_samples).flatten() + s_scaled = lb_vals[i] + s_unit * (ub_vals[i] - lb_vals[i]) + per_dim_samples.append(s_scaled.tolist()) + + candidate_points = list(product(*per_dim_samples)) + candidate_fims = [ + expected_obj._compute_fim_at_point_no_prior(0, list(pt)) + for pt in candidate_points + ] + + best_combo = None + best_obj = -np.inf + for combo in combinations(range(len(candidate_points)), n_exp): + fim_total = sum((candidate_fims[idx] for idx in combo), np.zeros((2, 2))) + obj_val = expected_obj._evaluate_objective_from_fim(fim_total) + if obj_val > best_obj: + best_obj = obj_val + best_combo = combo + + expected_points = [list(candidate_points[i]) for i in best_combo] + + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=n_exp) + actual_points, lhs_diag = doe._lhs_initialize_experiments( + lhs_n_samples=lhs_n_samples, + lhs_seed=lhs_seed, + n_exp=n_exp, + lhs_parallel=False, + lhs_combo_parallel=False, + ) + + actual_points_norm = sorted(tuple(np.round(p, 8)) for p in actual_points) + expected_points_norm = sorted(tuple(np.round(p, 8)) for p in expected_points) + self.assertEqual(actual_points_norm, expected_points_norm) + self.assertAlmostEqual(lhs_diag["best_obj"], best_obj, places=12) + + def test_optimize_experiments_is_reentrant_on_same_object(self): + # Tests that optimize_experiments() can be run repeatedly on one DoE object. + doe = self._make_template_doe("pseudo_trace") + doe.optimize_experiments(n_exp=1) + first_design = _optimize_experiments_param_scenario(doe.results)["experiments"][ + 0 + ]["design"] + first_build_time = doe.results["timing"]["build_time_s"] + + doe.optimize_experiments(n_exp=1) + second_design = _optimize_experiments_param_scenario(doe.results)[ + "experiments" + ][0]["design"] + + self.assertEqual(len(first_design), len(second_design)) + self.assertIn("timing", doe.results) + self.assertGreater(doe.results["timing"]["build_time_s"], 0.0) + self.assertGreaterEqual(first_build_time, 0.0) + self.assertEqual(len(list(doe.model.param_scenario_blocks.keys())), 1) + + def test_lhs_combo_parallel_matches_serial(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + # Use deterministic synthetic FIMs to isolate combo scorer behavior. + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, 2.0 * x + 1.0]]) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_serial, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, lhs_seed=123, n_exp=2, lhs_combo_parallel=False + ) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_parallel, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, + lhs_seed=123, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=2, + lhs_combo_parallel_threshold=1, + ) + + serial_norm = sorted(tuple(np.round(p, 8)) for p in points_serial) + parallel_norm = sorted(tuple(np.round(p, 8)) for p in points_parallel) + self.assertEqual(serial_norm, parallel_norm) + + def test_lhs_parallel_fim_eval_matches_serial(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + # Patch at class level so both serial path (self) and parallel worker + # DOE objects use the same deterministic synthetic FIM mapping. + def _fake_fim(self_obj, experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 0.5, 0.0], [0.0, 3.0 * x + 0.5]]) + + with patch.object( + DesignOfExperiments, + "_compute_fim_at_point_no_prior", + autospec=True, + side_effect=_fake_fim, + ): + points_serial, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, lhs_seed=321, n_exp=2, lhs_parallel=False + ) + + points_parallel, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, + lhs_seed=321, + n_exp=2, + lhs_parallel=True, + lhs_n_workers=2, + ) + + serial_norm = sorted(tuple(np.round(p, 8)) for p in points_serial) + parallel_norm = sorted(tuple(np.round(p, 8)) for p in points_parallel) + self.assertEqual(serial_norm, parallel_norm) + + def test_lhs_parallel_fim_eval_real_path_smoke(self): + doe_serial = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe_serial, n_exp=2) + points_serial, _ = doe_serial._lhs_initialize_experiments( + lhs_n_samples=3, + lhs_seed=9, + n_exp=2, + lhs_parallel=False, + lhs_combo_parallel=False, + ) + + doe_parallel = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe_parallel, n_exp=2) + points_parallel, _ = doe_parallel._lhs_initialize_experiments( + lhs_n_samples=3, + lhs_seed=9, + n_exp=2, + lhs_parallel=True, + lhs_n_workers=2, + lhs_combo_parallel=False, + ) + + serial_norm = sorted(tuple(np.round(p, 8)) for p in points_serial) + parallel_norm = sorted(tuple(np.round(p, 8)) for p in points_parallel) + self.assertEqual(serial_norm, parallel_norm) + + def test_lhs_parallel_solver_without_name_uses_default_worker_solver(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + doe.solver = object() + + def _fake_fim(self_obj, experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 0.5, 0.0], [0.0, x + 1.0]]) + + with patch.object( + DesignOfExperiments, + "_compute_fim_at_point_no_prior", + autospec=True, + side_effect=_fake_fim, + ): + with self.assertLogs("pyomo.contrib.doe.doe", level="DEBUG") as log_cm: + points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, + lhs_seed=14, + n_exp=2, + lhs_parallel=True, + lhs_n_workers=2, + lhs_combo_parallel=False, + ) + + self.assertEqual(len(points), 2) + self.assertTrue( + any("solver has no 'name' attribute" in msg for msg in log_cm.output) ) - # Verify that the linear scale plots were also created - # Check that we found exactly 5 files (A, D, E, ME, pseudo_A) - expected_plot_linear = glob(f"{prefix_linear}*.png") + def test_lhs_parallel_solver_factory_failure_uses_default_worker_solver(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + class _NamedSolver: + name = "definitely_missing_solver" + options = {} + + doe.solver = _NamedSolver() + + def _fake_fim(self_obj, experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 0.5, 0.0], [0.0, x + 1.0]]) + + with patch("pyomo.contrib.doe.doe.pyo.SolverFactory", return_value=None): + with patch.object( + DesignOfExperiments, + "_compute_fim_at_point_no_prior", + autospec=True, + side_effect=_fake_fim, + ): + with self.assertLogs("pyomo.contrib.doe.doe", level="DEBUG") as log_cm: + points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, + lhs_seed=18, + n_exp=2, + lhs_parallel=True, + lhs_n_workers=2, + lhs_combo_parallel=False, + ) + + self.assertEqual(len(points), 2) + self.assertTrue( + any( + "could not construct solver 'definitely_missing_solver'" in msg + for msg in log_cm.output + ) + ) + + def test_lhs_parallel_worker_exception_uses_zero_fim_fallback(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + with patch.object( + DesignOfExperiments, + "_compute_fim_at_point_no_prior", + side_effect=RuntimeError("worker boom"), + ): + with self.assertLogs("pyomo.contrib.doe.doe", level="ERROR") as log_cm: + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=4, + lhs_seed=22, + n_exp=2, + lhs_parallel=True, + lhs_n_workers=2, + lhs_combo_parallel=False, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(diag["timed_out"] is False) + self.assertTrue( + any( + "Using zero FIM for this candidate and continuing." in msg + for msg in log_cm.output + ) + ) + + def test_lhs_parallel_candidate_timeout_cancels_pending(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + # Track every submitted future so we can later verify which pending + # evaluations were explicitly cancelled when the deadline is exceeded. + created_futures = [] + + class _FakeFuture: + def __init__(self, idx): + self.idx = idx + self.cancelled = False + + def result(self): + # Return a deterministic candidate-FIM payload keyed by submit order. + x = float(self.idx + 1.0) + return self.idx, np.array([[x, 0.0], [0.0, x + 1.0]]) + + def cancel(self): + self.cancelled = True + return True + + class _FakeExecutor: + # Record submitted work but do not execute it asynchronously; the + # test controls completion order through the patched wait() call. + def __init__(self, max_workers=None): + self.created = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def submit(self, fn, idx_pt): + fut = _FakeFuture(idx_pt[0]) + self.created.append(fut) + created_futures.append(fut) + return fut + + def _fake_wait(pending, timeout=None, return_when=None): + # Pretend exactly one future finishes per wait cycle and leave the + # rest pending so timeout handling has something to cancel. + done = {min(pending, key=lambda fut: fut.idx)} + still_pending = set(pending) - done + return done, still_pending + + perf_counter_calls = {"n": 0} + + def _fake_perf_counter(): + # Keep the clock at t=0 long enough to fill the initial pending + # queue, then jump past the deadline so the code cancels leftovers. + perf_counter_calls["n"] += 1 + if perf_counter_calls["n"] <= 6: + return 0.0 + return 1.0 + + with patch("pyomo.contrib.doe.doe._cf.ThreadPoolExecutor", _FakeExecutor): + with patch("pyomo.contrib.doe.doe._cf.wait", side_effect=_fake_wait): + with patch( + "pyomo.contrib.doe.doe.time.perf_counter", + side_effect=_fake_perf_counter, + ): + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=20, + lhs_seed=5, + n_exp=2, + lhs_parallel=True, + lhs_n_workers=2, + lhs_combo_parallel=False, + lhs_max_wall_clock_time=0.5, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(diag["timed_out"]) + # With 2 workers, the implementation allows up to 4 pending candidate + # FIM futures before waiting for one to complete. + self.assertEqual(len(created_futures), 4) + # Our fake scheduler completes futures 0 and 1, so timeout should only + # cancel the still-pending futures from the tail of that first batch. + self.assertEqual(sum(fut.cancelled for fut in created_futures), 2) + self.assertEqual([fut.idx for fut in created_futures if fut.cancelled], [2, 3]) + + def test_lhs_no_candidate_fim_scored_uses_zero_fallback(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=1) + + with patch.object(doe, "_compute_fim_at_point_no_prior", return_value=None): + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=3, + lhs_seed=8, + n_exp=1, + lhs_parallel=False, + lhs_combo_parallel=False, + ) + + self.assertEqual(len(points), 1) + self.assertTrue(diag["timed_out"]) + + def test_lhs_candidate_subset_padding_to_n_exp(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=3) + call_state = {"n": 0} + + def _fim_with_one_slow_call(experiment_index, input_values): + call_state["n"] += 1 + x = float(input_values[0]) + if call_state["n"] == 2: + time.sleep(0.1) + return np.array([[x + 1.0, 0.0], [0.0, x + 2.0]]) + + with patch.object( + doe, "_compute_fim_at_point_no_prior", side_effect=_fim_with_one_slow_call + ): + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=5, + lhs_seed=16, + n_exp=3, + lhs_parallel=False, + lhs_combo_parallel=False, + lhs_max_wall_clock_time=0.05, + ) + + self.assertEqual(len(points), 3) + self.assertTrue(diag["timed_out"]) + self.assertLess(diag["n_candidates"], 5) + + def test_lhs_combo_parallel_chunk_boundary_matches_serial(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 2.0, 0.0], [0.0, x + 1.0]]) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_serial, _ = doe._lhs_initialize_experiments( + lhs_n_samples=5, lhs_seed=99, n_exp=2, lhs_combo_parallel=False + ) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_parallel, _ = doe._lhs_initialize_experiments( + lhs_n_samples=5, + lhs_seed=99, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=3, # C(5,2)=10, non-divisible chunking + lhs_combo_parallel_threshold=1, + ) + + serial_norm = sorted(tuple(np.round(p, 8)) for p in points_serial) + parallel_norm = sorted(tuple(np.round(p, 8)) for p in points_parallel) + self.assertEqual(serial_norm, parallel_norm) + + def test_lhs_combo_parallel_warning_reports_single_worker_reason(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + with self.assertLogs("pyomo.contrib.doe.doe", level="WARNING") as log_cm: + points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, + lhs_seed=27, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=1, + lhs_combo_parallel_threshold=1, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(any("resolved_workers=1 <= 1" in msg for msg in log_cm.output)) + + def test_lhs_combo_parallel_skips_empty_local_combo(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + with patch.object( + DesignOfExperiments, + "_evaluate_objective_for_option", + return_value=float("nan"), + ): + points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=4, + lhs_seed=33, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=2, + lhs_combo_parallel_threshold=1, + ) + + self.assertEqual(len(points), 2) + + def test_lhs_combo_scoring_timeout_returns_best_so_far(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + def _slow_obj(fim, objective_option): + time.sleep(0.003) + return float(np.trace(fim)) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + with patch.object( + DesignOfExperiments, + "_evaluate_objective_for_option", + side_effect=_slow_obj, + ): + with self.assertLogs( + "pyomo.contrib.doe.doe", level="WARNING" + ) as log_cm: + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=6, + lhs_seed=7, + n_exp=2, + lhs_combo_parallel=False, + lhs_max_wall_clock_time=0.001, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(any("time budget" in msg for msg in log_cm.output)) + self.assertTrue(diag["timed_out"]) + + def test_lhs_combo_scoring_parallel_timeout_returns_best_so_far(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + def _slow_obj(fim, objective_option): + time.sleep(0.003) + return float(np.trace(fim)) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + with patch.object( + DesignOfExperiments, + "_evaluate_objective_for_option", + side_effect=_slow_obj, + ): + with self.assertLogs( + "pyomo.contrib.doe.doe", level="WARNING" + ) as log_cm: + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=6, + lhs_seed=12, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=5, + lhs_combo_parallel_threshold=1, + lhs_max_wall_clock_time=0.001, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(any("time budget" in msg for msg in log_cm.output)) + self.assertTrue(diag["timed_out"]) + + def test_lhs_combo_parallel_timeout_cancels_pending(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + def _slow_obj(fim, objective_option): + time.sleep(0.01) + return float(np.trace(fim)) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + with patch.object( + DesignOfExperiments, + "_evaluate_objective_for_option", + side_effect=_slow_obj, + ): + with self.assertLogs( + "pyomo.contrib.doe.doe", level="WARNING" + ) as log_cm: + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=9, + lhs_seed=4, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=1, + lhs_combo_parallel_threshold=1, + lhs_max_wall_clock_time=0.02, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(diag["timed_out"]) + self.assertTrue(any("time budget" in msg for msg in log_cm.output)) + + def test_lhs_combo_parallel_submit_loop_timeout_sets_timed_out(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=1) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + times = iter([0.0, 0.0, 0.0, 0.0, 0.01, 0.01, 0.01]) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + with patch( + "pyomo.contrib.doe.doe.time.perf_counter", + side_effect=lambda: next(times), + ): + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=1, + lhs_seed=19, + n_exp=1, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=1, + lhs_combo_parallel_threshold=1, + lhs_max_wall_clock_time=0.001, + ) + + self.assertEqual(len(points), 1) + self.assertTrue(diag["timed_out"]) + + def test_lhs_combo_parallel_deadline_cancels_pending_futures(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + stage = {"timeout": False} + created_futures = [] + + class _FakeFuture: + def __init__(self, idx, payload): + self.idx = idx + self._payload = payload + self.cancelled = False + + def result(self): + return self._payload + + def cancel(self): + self.cancelled = True + return True + + class _FakeExecutor: + def __init__(self, max_workers=None): + self.created = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def submit(self, fn, *args, **kwargs): + # Return deterministic chunk scores; first future will be "done". + idx = len(self.created) + payload = ( + (10.0 - idx, (0, 1), False) + if idx == 0 + else (9.0 - idx, (0, 2), False) + ) + fut = _FakeFuture(idx, payload) + self.created.append(fut) + created_futures.append(fut) + return fut + + def _fake_wait(pending, return_when=None): + done = {min(pending, key=lambda fut: fut.idx)} + still_pending = set(pending) - done + stage["timeout"] = True + return done, still_pending + + def _fake_perf_counter(): + return 1.0 if stage["timeout"] else 0.0 + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + with patch("pyomo.contrib.doe.doe._cf.ThreadPoolExecutor", _FakeExecutor): + with patch("pyomo.contrib.doe.doe._cf.wait", side_effect=_fake_wait): + with patch( + "pyomo.contrib.doe.doe.time.perf_counter", + side_effect=_fake_perf_counter, + ): + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=3, + lhs_seed=23, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=1, + lhs_combo_parallel_threshold=1, + lhs_max_wall_clock_time=0.5, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(diag["timed_out"]) + self.assertEqual(len(created_futures), 3) + self.assertEqual(sum(fut.cancelled for fut in created_futures), 2) + self.assertEqual([fut.idx for fut in created_futures if fut.cancelled], [1, 2]) + + def test_lhs_combo_parallel_minimize_objective_update(self): + doe = self._make_template_doe("trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, 3.0 * x + 1.0]]) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_serial, _ = doe._lhs_initialize_experiments( + lhs_n_samples=5, lhs_seed=52, n_exp=2, lhs_combo_parallel=False + ) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_parallel, _ = doe._lhs_initialize_experiments( + lhs_n_samples=5, + lhs_seed=52, + n_exp=2, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=2, + lhs_combo_parallel_threshold=1, + ) + + serial_norm = sorted(tuple(np.round(p, 8)) for p in points_serial) + parallel_norm = sorted(tuple(np.round(p, 8)) for p in points_parallel) + self.assertEqual(serial_norm, parallel_norm) + + def test_lhs_combo_serial_minimize_objective_update(self): + doe = self._make_template_doe("trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + lhs_n_samples = 4 + lhs_seed = 61 + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 0.25, 0.0], [0.0, 2.5 * x + 0.25]]) + + first_exp_block = doe.model.param_scenario_blocks[0].exp_blocks[0] + exp_input_vars = doe._get_experiment_input_vars(first_exp_block) + lb_vals = np.array([v.lb for v in exp_input_vars]) + ub_vals = np.array([v.ub for v in exp_input_vars]) + + rng = np.random.default_rng(lhs_seed) + from scipy.stats.qmc import LatinHypercube + + per_dim_samples = [] + for i in range(len(exp_input_vars)): + dim_seed = int(rng.integers(0, 2**31)) + sampler = LatinHypercube(d=1, seed=dim_seed) + s_unit = sampler.random(n=lhs_n_samples).flatten() + s_scaled = lb_vals[i] + s_unit * (ub_vals[i] - lb_vals[i]) + per_dim_samples.append(s_scaled.tolist()) + candidate_points = list(product(*per_dim_samples)) + candidate_fims = [_fake_fim(0, pt) for pt in candidate_points] + + best_combo = None + best_obj = np.inf + for combo in combinations(range(len(candidate_points)), 2): + fim_total = sum((candidate_fims[idx] for idx in combo), np.zeros((2, 2))) + obj_val = doe._evaluate_objective_from_fim(fim_total) + if obj_val < best_obj: + best_obj = obj_val + best_combo = combo + expected_points = [list(candidate_points[i]) for i in best_combo] + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + got_points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=lhs_n_samples, + lhs_seed=lhs_seed, + n_exp=2, + lhs_combo_parallel=False, + ) + + got_norm = sorted(tuple(np.round(p, 8)) for p in got_points) + exp_norm = sorted(tuple(np.round(p, 8)) for p in expected_points) + self.assertEqual(got_norm, exp_norm) + + def test_lhs_combo_no_scored_combo_falls_back_to_first_n_exp(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=3) + lhs_n_samples = 3 + unit_samples = np.array([0.2, 0.5, 0.8]) + + class _FakeLHS: + def __init__(self, d, seed=None): + self.d = d + + def random(self, n): + assert self.d == 1 + assert n == lhs_n_samples + return unit_samples.reshape((n, 1)) + + first_exp_block = doe.model.param_scenario_blocks[0].exp_blocks[0] + exp_input_vars = doe._get_experiment_input_vars(first_exp_block) + lb = float(exp_input_vars[0].lb) + ub = float(exp_input_vars[0].ub) + expected_points = [[float(lb + s * (ub - lb))] for s in unit_samples] + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 1.0]]) + + with patch("pyomo.contrib.doe.doe.LatinHypercube", _FakeLHS): + with patch("pyomo.contrib.doe.doe._combinations", return_value=iter(())): + with patch.object( + doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim + ): + with self.assertLogs( + "pyomo.contrib.doe.doe", level="WARNING" + ) as log_cm: + got_points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=lhs_n_samples, + lhs_seed=13, + n_exp=3, + lhs_combo_parallel=False, + ) + + got_norm = sorted(tuple(np.round(p, 8)) for p in got_points) + exp_norm = sorted(tuple(np.round(p, 8)) for p in expected_points) + self.assertEqual(got_norm, exp_norm) self.assertTrue( - len(expected_plot_linear) == 5, - f"Expected 5 plot files, but found {len(expected_plot_linear)}. Files found: {expected_plot_linear}", + any( + "Falling back to the first n_exp candidate points." in msg + for msg in log_cm.output + ) + ) + + def test_lhs_fim_evaluation_timeout_stops_early(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + + def _slow_fim(experiment_index, input_values): + time.sleep(0.01) + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, x + 2.0]]) + + with patch.object( + doe, "_compute_fim_at_point_no_prior", side_effect=_slow_fim + ) as p: + points, diag = doe._lhs_initialize_experiments( + lhs_n_samples=10, + lhs_seed=3, + n_exp=2, + lhs_parallel=False, + lhs_combo_parallel=False, + lhs_max_wall_clock_time=0.03, + ) + + self.assertEqual(len(points), 2) + self.assertTrue(diag["timed_out"]) + self.assertLess(p.call_count, 10) + self.assertLess(diag["n_candidates"], 10) + + def test_lhs_combo_scoring_n_exp_3_parallel_matches_serial(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=3) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, 0.5 * x + 2.0]]) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_serial, _ = doe._lhs_initialize_experiments( + lhs_n_samples=5, lhs_seed=1234, n_exp=3, lhs_combo_parallel=False + ) + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + points_parallel, _ = doe._lhs_initialize_experiments( + lhs_n_samples=5, + lhs_seed=1234, + n_exp=3, + lhs_combo_parallel=True, + lhs_n_workers=2, + lhs_combo_chunk_size=3, + lhs_combo_parallel_threshold=1, + ) + + serial_norm = sorted(tuple(np.round(p, 8)) for p in points_serial) + parallel_norm = sorted(tuple(np.round(p, 8)) for p in points_parallel) + self.assertEqual(serial_norm, parallel_norm) + self.assertEqual(len(points_serial), 3) + + def test_lhs_combo_scoring_n_exp_3_matches_oracle(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=3) + lhs_n_samples = 5 + lhs_seed = 2 + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, 2.0 * x + 0.5]]) + + # Recreate the exact candidate points from LHS generation (independent + # oracle for combination scoring logic). + first_exp_block = doe.model.param_scenario_blocks[0].exp_blocks[0] + exp_input_vars = doe._get_experiment_input_vars(first_exp_block) + lb_vals = np.array([v.lb for v in exp_input_vars]) + ub_vals = np.array([v.ub for v in exp_input_vars]) + rng = np.random.default_rng(lhs_seed) + from scipy.stats.qmc import LatinHypercube + + per_dim_samples = [] + for i in range(len(exp_input_vars)): + dim_seed = int(rng.integers(0, 2**31)) + sampler = LatinHypercube(d=1, seed=dim_seed) + s_unit = sampler.random(n=lhs_n_samples).flatten() + s_scaled = lb_vals[i] + s_unit * (ub_vals[i] - lb_vals[i]) + per_dim_samples.append(s_scaled.tolist()) + candidate_points = list(product(*per_dim_samples)) + + # Oracle over all combinations of size 3. + fims = [_fake_fim(0, pt) for pt in candidate_points] + best_obj = -np.inf + best_combo = None + for combo in combinations(range(len(candidate_points)), 3): + fim_total = sum((fims[i] for i in combo), np.zeros((2, 2))) + obj = float(np.trace(fim_total)) + if obj > best_obj: + best_obj = obj + best_combo = combo + expected_points = [list(candidate_points[i]) for i in best_combo] + + with patch.object(doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim): + got_points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=lhs_n_samples, + lhs_seed=lhs_seed, + n_exp=3, + lhs_combo_parallel=False, + ) + + got_norm = sorted(tuple(np.round(p, 8)) for p in got_points) + exp_norm = sorted(tuple(np.round(p, 8)) for p in expected_points) + self.assertEqual(got_norm, exp_norm) + + def test_lhs_matches_independent_oracle_with_fixed_samples(self): + doe = self._make_template_doe("pseudo_trace") + self._build_template_model_for_multi_experiment(doe, n_exp=2) + lhs_n_samples = 3 + + first_exp_block = doe.model.param_scenario_blocks[0].exp_blocks[0] + exp_input_vars = doe._get_experiment_input_vars(first_exp_block) + self.assertEqual(len(exp_input_vars), 1) + lb = float(exp_input_vars[0].lb) + ub = float(exp_input_vars[0].ub) + + unit_samples = np.array([0.1, 0.4, 0.9]) + scaled_points = [float(lb + s * (ub - lb)) for s in unit_samples] + + class _FakeLHS: + def __init__(self, d, seed=None): + self.d = d + + def random(self, n): + assert self.d == 1 + assert n == lhs_n_samples + return unit_samples.reshape((n, 1)) + + def _fake_fim(experiment_index, input_values): + x = float(input_values[0]) + return np.array([[x + 1.0, 0.0], [0.0, 2.0 * x + 1.0]]) + + with patch("pyomo.contrib.doe.doe.LatinHypercube", _FakeLHS): + with patch.object( + doe, "_compute_fim_at_point_no_prior", side_effect=_fake_fim + ): + got_points, _ = doe._lhs_initialize_experiments( + lhs_n_samples=lhs_n_samples, + lhs_seed=123, + n_exp=2, + lhs_combo_parallel=False, + ) + + # Independent brute-force oracle over explicit candidate points + best_obj = -np.inf + best_combo = None + for combo in combinations(range(len(scaled_points)), 2): + f1 = _fake_fim(0, [scaled_points[combo[0]]]) + f2 = _fake_fim(0, [scaled_points[combo[1]]]) + obj_val = float(np.trace(f1 + f2)) + if obj_val > best_obj: + best_obj = obj_val + best_combo = combo + expected_points = [[scaled_points[i]] for i in best_combo] + + got_norm = sorted(tuple(np.round(p, 8)) for p in got_points) + exp_norm = sorted(tuple(np.round(p, 8)) for p in expected_points) + self.assertEqual(got_norm, exp_norm) + + def test_optimize_experiments_determinant_expected_values(self): + # Tests determinant-objective optimization against known expected design/metric values. + # Match the multi-experiment example style (explicit experiment list) + exp_list = [ + RooneyBieglerMultiExperiment(hour=1.0, y=8.3), + RooneyBieglerMultiExperiment(hour=2.0, y=10.3), + ] + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="determinant", + step=1e-2, + solver=solver, + ) + doe.optimize_experiments() + + scenario = _optimize_experiments_param_scenario(doe.results) + got_hours = sorted(exp["design"][0] for exp in scenario["experiments"]) + expected_hours = [1.9321985035514362, 9.999999685577139] + + self.assertStructuredAlmostEqual(got_hours, expected_hours, abstol=1e-3) + self.assertAlmostEqual( + scenario["quality_metrics"]["log10_d_opt"], 6.028152580313302, places=3 + ) + + def test_optimize_experiments_trace_expected_values(self): + # Tests trace-objective optimization against known expected design/metric values. + # Match the multi-experiment example style (explicit experiment list) + exp_list = [ + RooneyBieglerMultiExperiment(hour=1.0, y=8.3), + RooneyBieglerMultiExperiment(hour=2.0, y=10.3), + ] + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + # prior_FIM from data `hour = 1, y = 8.3` with default value of parameters, which + # is theta = {'asymptote': 15, 'rate_constant': 0.5} + prior_FIM = np.array( + [[15.48181217, 357.97684273], [357.97684273, 8277.28811613]] + ) + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="trace", + step=1e-2, + solver=solver, + prior_FIM=prior_FIM, + ) + doe.optimize_experiments() + + scenario = _optimize_experiments_param_scenario(doe.results) + got_hours = sorted(exp["design"][0] for exp in scenario["experiments"]) + expected_hours = [10.0, 10.0] + + self.assertStructuredAlmostEqual(got_hours, expected_hours, abstol=1e-3) + self.assertAlmostEqual( + scenario["quality_metrics"]["log10_a_opt"], -2.2347, places=3 ) - # Run the reactor example with log scale - run_reactor_doe( - n_points_for_C0=1, - n_points_for_T0=1, - compute_FIM_full_factorial=True, - plot_factorial_results=True, - figure_file_name=prefix_log, - log_scale=True, - run_optimal_doe=False, + def test_optimize_experiments_prior_fim_aggregation_non_lhs_template_mode(self): + # Tests that total FIM equals sum(experiment FIMs) + prior in template mode. + prior_fim = np.array([[2.0, 0.1], [0.1, 1.5]]) + doe = self._make_template_doe("pseudo_trace") + doe.prior_FIM = prior_fim.copy() + + doe.optimize_experiments(n_exp=2, init_method=None) + + scenario = _optimize_experiments_param_scenario(doe.results) + total_fim = np.array(scenario["total_fim"]) + exp_fim_sum = sum( + (np.array(exp_data["fim"]) for exp_data in scenario["experiments"]), + np.zeros_like(total_fim), ) + stored_prior = np.array(doe.results["problem"]["prior_fim"]) + + self.assertTrue(np.allclose(total_fim, exp_fim_sum + prior_fim, atol=1e-6)) + self.assertTrue(np.allclose(total_fim, exp_fim_sum + stored_prior, atol=1e-6)) + + def test_optimize_experiments_prior_fim_aggregation_non_lhs_user_initialized_mode( + self, + ): + # Tests that total FIM aggregation with prior is correct in user-initialized mode. + exp_list = [ + RooneyBieglerMultiExperiment(hour=1.5, y=9.0), + RooneyBieglerMultiExperiment(hour=3.5, y=12.0), + ] + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + prior_fim = np.array([[1.25, 0.05], [0.05, 0.9]]) + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="pseudo_trace", + step=1e-2, + solver=solver, + prior_FIM=prior_fim, + ) + doe.optimize_experiments(init_method=None) - # Verify that the log scale plots were also created - # Check that we found exactly 5 files (A, D, E, ME, pseudo_A) - expected_plot_log = glob(f"{prefix_log}*.png") + scenario = _optimize_experiments_param_scenario(doe.results) + total_fim = np.array(scenario["total_fim"]) + exp_fim_sum = sum( + (np.array(exp_data["fim"]) for exp_data in scenario["experiments"]), + np.zeros_like(total_fim), + ) + stored_prior = np.array(doe.results["problem"]["prior_fim"]) + + self.assertTrue(np.allclose(total_fim, exp_fim_sum + prior_fim, atol=1e-6)) + self.assertTrue(np.allclose(total_fim, exp_fim_sum + stored_prior, atol=1e-6)) + + def test_optimize_experiments_safe_metric_failure_sets_nan(self): + # Tests that metric-computation failures are captured as NaN with a warning. + doe = self._make_template_doe("pseudo_trace") + with patch( + "pyomo.contrib.doe.doe.np.linalg.inv", side_effect=RuntimeError("boom") + ): + with self.assertLogs("pyomo.contrib.doe.doe", level="WARNING") as log_cm: + doe.optimize_experiments(n_exp=1) + + scenario = _optimize_experiments_param_scenario(doe.results) + self.assertTrue(np.isnan(scenario["quality_metrics"]["log10_a_opt"])) self.assertTrue( - len(expected_plot_log) == 5, - f"Expected 5 plot files, but found {len(expected_plot_log)}. Files found: {expected_plot_log}", + any("failed to compute log10 A-opt" in msg for msg in log_cm.output) + ) + + def test_optimize_experiments_non_cholesky_determinant_initialization(self): + # Tests determinant initialization correctness when Cholesky formulation is disabled. + exp = RooneyBieglerMultiExperiment(hour=2.0, y=10.0) + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + doe = DesignOfExperiments( + experiment=[exp], + objective_option="determinant", + step=1e-2, + solver=solver, + _Cholesky_option=False, + _only_compute_fim_lower=False, + ) + original_solve = doe.solver.solve + + class _MockSolverInfo: + status = "ok" + termination_condition = "optimal" + message = "mock-solve" + + class _MockResults: + solver = _MockSolverInfo() + + def _solve_real_for_square_then_mock(*args, **kwargs): + model = args[0] if args else kwargs.get("model", None) + if model is not None and hasattr(model, "dummy_obj"): + # Keep square-solve path real so model state initializes correctly. + return original_solve(*args, **kwargs) + return _MockResults() + + with patch.object( + doe.solver, "solve", side_effect=_solve_real_for_square_then_mock + ): + doe.optimize_experiments(n_exp=1) + + scenario_block = doe.model.param_scenario_blocks[0] + self.assertTrue(hasattr(scenario_block.obj_cons, "determinant")) + total_fim = np.array( + _optimize_experiments_param_scenario(doe.results)["total_fim"] + ) + expected_det = np.linalg.det(total_fim) + self.assertAlmostEqual( + pyo.value(scenario_block.obj_cons.determinant), expected_det, places=6 ) diff --git a/pyomo/contrib/doe/tests/test_greybox.py b/pyomo/contrib/doe/tests/test_greybox.py index 2fd007e6d52..c7294710571 100644 --- a/pyomo/contrib/doe/tests/test_greybox.py +++ b/pyomo/contrib/doe/tests/test_greybox.py @@ -8,8 +8,7 @@ # ____________________________________________________________________________________ import copy import itertools -import json -import os.path +from unittest.mock import patch from pyomo.common.dependencies import ( numpy as np, @@ -19,20 +18,21 @@ scipy_available, ) -from pyomo.common.fileutils import this_file_dir import pyomo.common.unittest as unittest if not (numpy_available and scipy_available): raise unittest.SkipTest("Pyomo.DoE needs scipy and numpy to run tests") -if scipy_available: - from pyomo.contrib.doe import DesignOfExperiments, FIMExternalGreyBox - from pyomo.contrib.doe.examples.reactor_example import ( - ReactorExperiment as FullReactorExperiment, - ) - from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( - RooneyBieglerExperiment, - ) +from pyomo.contrib.doe import DesignOfExperiments, FIMExternalGreyBox +from pyomo.contrib.doe.examples.reactor_example import ( + ReactorExperiment as FullReactorExperiment, +) +from pyomo.contrib.doe.tests.experiment_class_example_flags import ( + RooneyBieglerMultiExperiment, +) +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) import pyomo.environ as pyo @@ -41,27 +41,16 @@ ipopt_available = SolverFactory("ipopt").available() cyipopt_available = SolverFactory("cyipopt").available() -currdir = this_file_dir() -file_path = os.path.join(currdir, "..", "examples", "result.json") - -with open(file_path) as f: - data_ex = json.load(f) - -data_ex["control_points"] = {float(k): v for k, v in data_ex["control_points"].items()} - _FD_EPSILON_FIRST = 1e-6 # Epsilon for numerical comparison of derivatives _FD_EPSILON_SECOND = 1e-4 # Epsilon for numerical comparison of derivatives if numpy_available: # Randomly generated P.S.D. matrix - # Matrix is 4x4 to match example - # number of parameters. + # Matrix is 2x2 to match Rooney-Biegler parameters. testing_matrix = np.array( [ - [5.13730123, 1.08084953, 1.6466824, 1.09943223], - [1.08084953, 1.57183404, 1.50704403, 1.4969689], - [1.6466824, 1.50704403, 2.54754738, 1.39902838], - [1.09943223, 1.4969689, 1.39902838, 1.57406692], + [5.13730123, 1.08084953], + [1.08084953, 1.57183404], ] ) @@ -203,18 +192,17 @@ def get_numerical_second_derivative(grey_box_object=None, return_reduced=True): numerical_derivative[i, j, k, l] = diff if return_reduced: - # This considers a 4-parameter system - # which is what these tests are based - # upon. This can be generalized but - # requires checking the parameter length. - # # Make ordered quads with no repeats # of the ordered pairs - ordered_pairs = itertools.product(range(4), repeat=2) - ordered_pairs_list = list(itertools.combinations_with_replacement(range(4), 2)) + ordered_pairs = itertools.product(range(dim), repeat=2) + ordered_pairs_list = list( + itertools.combinations_with_replacement(range(dim), 2) + ) ordered_quads = itertools.combinations_with_replacement(ordered_pairs, 2) - numerical_derivative_reduced = np.zeros((10, 10)) + numerical_derivative_reduced = np.zeros( + (len(ordered_pairs_list), len(ordered_pairs_list)) + ) for curr_quad in ordered_quads: d1, d2 = curr_quad @@ -279,23 +267,7 @@ def get_standard_args(experiment, fd_method, obj_used): def make_greybox_and_doe_objects(objective_option): - fd_method = "central" - obj_used = objective_option - - experiment = FullReactorExperiment(data_ex, 10, 3) - - DoE_args = get_standard_args(experiment, fd_method, obj_used) - DoE_args["use_grey_box_objective"] = True - DoE_args["prior_FIM"] = testing_matrix - - doe_obj = DesignOfExperiments(**DoE_args) - doe_obj.create_doe_model() - - grey_box_object = FIMExternalGreyBox( - doe_object=doe_obj, objective_option=doe_obj.objective_option - ) - - return doe_obj, grey_box_object + return make_greybox_and_doe_objects_rooney_biegler(objective_option) def make_greybox_and_doe_objects_rooney_biegler(objective_option): @@ -346,11 +318,128 @@ def make_greybox_and_doe_objects_rooney_biegler(objective_option): return doe_obj, grey_box_object +class _MockGreyBoxSolver: + def __init__(self, name="mock-greybox"): + self.name = name + self.calls = [] + + def solve(self, model, tee=False): + self.calls.append({"model": model, "tee": tee}) + + class _MockSolverInfo: + status = "ok" + termination_condition = "optimal" + message = "mock-greybox-solve" + + class _MockResults: + solver = _MockSolverInfo() + + return _MockResults() + + +def _make_ipopt_solver(): + solver = SolverFactory("ipopt") + solver.options["linear_solver"] = "ma57" + solver.options["halt_on_ampl_error"] = "yes" + solver.options["max_iter"] = 3000 + return solver + + +def _make_cyipopt_solver(tol=1e-4): + grey_box_solver = SolverFactory("cyipopt") + grey_box_solver.config.options["linear_solver"] = "ma57" + grey_box_solver.config.options['tol'] = tol + grey_box_solver.config.options['mu_strategy'] = "monotone" + return grey_box_solver + + +def _make_multiexperiment_greybox_doe( + objective_option, prior_FIM=None, grey_box_solver=None +): + if prior_FIM is None: + prior_FIM = np.zeros((2, 2)) + return DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0, y=10.0)], + objective_option=objective_option, + use_grey_box_objective=True, + step=1e-2, + solver=_make_ipopt_solver(), + grey_box_solver=( + grey_box_solver if grey_box_solver is not None else _MockGreyBoxSolver() + ), + prior_FIM=prior_FIM, + ) + + +def _get_multiexperiment_scenario_data(doe_obj): + scenario = doe_obj.model.param_scenario_blocks[0] + total_fim = np.array(doe_obj.results["solution"]["param_scenarios"][0]["total_fim"]) + parameter_names = list(scenario.exp_blocks[0].parameter_names) + return scenario, total_fim, parameter_names + + +def _generate_lhs_candidate_points(doe_obj, lhs_n_samples, lhs_seed): + from scipy.stats.qmc import LatinHypercube + + first_exp_block = doe_obj.model.param_scenario_blocks[0].exp_blocks[0] + exp_input_vars = doe_obj._get_experiment_input_vars(first_exp_block) + lb_vals = np.array([v.lb for v in exp_input_vars]) + ub_vals = np.array([v.ub for v in exp_input_vars]) + + rng = np.random.default_rng(lhs_seed) + per_dim_samples = [] + for i in range(len(exp_input_vars)): + dim_seed = int(rng.integers(0, 2**31)) + sampler = LatinHypercube(d=1, seed=dim_seed) + s_unit = sampler.random(n=lhs_n_samples).flatten() + s_scaled = lb_vals[i] + s_unit * (ub_vals[i] - lb_vals[i]) + per_dim_samples.append(s_scaled.tolist()) + + return list(itertools.product(*per_dim_samples)) + + +def _expected_multiexperiment_greybox_output(objective_option, fim_np): + if objective_option == "trace": + return float(np.trace(np.linalg.pinv(fim_np))) + if objective_option == "determinant": + return float(np.log(np.linalg.det(fim_np))) + if objective_option == "minimum_eigenvalue": + return float(np.min(np.linalg.eigvalsh(fim_np))) + if objective_option == "condition_number": + eig = np.linalg.eigvalsh(fim_np) + return float(np.log(np.abs(np.max(eig) / np.min(eig)))) + raise AssertionError(f"Unexpected greybox objective: {objective_option!r}") + + +def _reconstruct_fim_from_egb_inputs(egb_block, parameter_names): + dim = len(parameter_names) + fim = np.zeros((dim, dim)) + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + if i >= j: + fim[i, j] = pyo.value(egb_block.inputs[(q, p)]) + fim[j, i] = fim[i, j] + return fim + + +def _spd_hour_fim_oracle(experiment_index, input_values): + hour = float(input_values[0]) + return np.array([[hour + 2.0, 0.2 * hour], [0.2 * hour, 14.0 - hour]]) + + +def _diagonal_hour_fim_oracle(experiment_index, input_values): + hour = float(input_values[0]) + return np.eye(2) * (hour + 1.0) + + +# Test whether the cyipopt GreyBox solve path can actually +# run with the MA57/HSL linear solver required by these tests. # Test whether or not cyipopt # is appropriately calling the # linear solvers. bad_message = "Invalid option encountered." cyipopt_call_working = True +cyipopt_skip_reason = "cyipopt GreyBox solve path requires a working MA57/HSL runtime" if ( numpy_available and scipy_available @@ -373,11 +462,20 @@ def make_greybox_and_doe_objects_rooney_biegler(objective_option): doe_object.run_doe() - cyipopt_call_working = not ( - bad_message in doe_object.results["Termination Message"] - ) - except: + termination_message = str(doe_object.results.get("Termination Message", "")) + cyipopt_call_working = bad_message not in termination_message + if not cyipopt_call_working: + # The GreyBox path here uses cyipopt rather than the standalone + # IPOPT executable, and the local failure mode we diagnosed was a + # missing MA57/HSL runtime on that cyipopt solve path. + cyipopt_skip_reason = ( + "cyipopt GreyBox solve path cannot access the MA57/HSL runtime" + ) + except Exception: cyipopt_call_working = False + cyipopt_skip_reason = ( + "cyipopt GreyBox solve path could not be initialized with MA57/HSL" + ) @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") @@ -412,16 +510,9 @@ def test_input_names(self): # Hard-coded names of the inputs, in the order we expect input_names = [ - ('A1', 'A1'), - ('A1', 'A2'), - ('A1', 'E1'), - ('A1', 'E2'), - ('A2', 'A2'), - ('A2', 'E1'), - ('A2', 'E2'), - ('E1', 'E1'), - ('E1', 'E2'), - ('E2', 'E2'), + ('asymptote', 'asymptote'), + ('asymptote', 'rate_constant'), + ('rate_constant', 'rate_constant'), ] # Grabbing input names from grey box object @@ -788,10 +879,8 @@ def test_A_opt_greybox_build(self): # Check to see if each component exists all_exist = True - # Check output and value - # FIM Initial will be the prior FIM - # added with the identity matrix. - A_opt_val = np.trace(np.linalg.inv(testing_matrix + np.eye(4))) + expected_FIM = _._get_FIM() + A_opt_val = np.trace(np.linalg.inv(expected_FIM)) try: A_opt_val_gb = doe_obj.model.obj_cons.egb_fim_block.outputs["A-opt"].value @@ -820,7 +909,7 @@ def test_A_opt_greybox_build(self): current_FIM[np.triu_indices_from(current_FIM)] = input_values current_FIM += current_FIM.transpose() - np.diag(np.diag(current_FIM)) - self.assertTrue(np.all(np.isclose(current_FIM, testing_matrix + np.eye(4)))) + self.assertTrue(np.all(np.isclose(current_FIM, expected_FIM))) def test_D_opt_greybox_build(self): objective_option = "determinant" @@ -833,10 +922,8 @@ def test_D_opt_greybox_build(self): # Check to see if each component exists all_exist = True - # Check output and value - # FIM Initial will be the prior FIM - # added with the identity matrix. - D_opt_val = np.log(np.linalg.det(testing_matrix + np.eye(4))) + expected_FIM = _._get_FIM() + D_opt_val = np.log(np.linalg.det(expected_FIM)) try: D_opt_val_gb = doe_obj.model.obj_cons.egb_fim_block.outputs[ @@ -867,7 +954,7 @@ def test_D_opt_greybox_build(self): current_FIM[np.triu_indices_from(current_FIM)] = input_values current_FIM += current_FIM.transpose() - np.diag(np.diag(current_FIM)) - self.assertTrue(np.all(np.isclose(current_FIM, testing_matrix + np.eye(4)))) + self.assertTrue(np.all(np.isclose(current_FIM, expected_FIM))) def test_E_opt_greybox_build(self): objective_option = "minimum_eigenvalue" @@ -880,10 +967,8 @@ def test_E_opt_greybox_build(self): # Check to see if each component exists all_exist = True - # Check output and value - # FIM Initial will be the prior FIM - # added with the identity matrix. - vals, vecs = np.linalg.eig(testing_matrix + np.eye(4)) + expected_FIM = _._get_FIM() + vals, vecs = np.linalg.eig(expected_FIM) E_opt_val = np.min(vals) try: @@ -913,7 +998,7 @@ def test_E_opt_greybox_build(self): current_FIM[np.triu_indices_from(current_FIM)] = input_values current_FIM += current_FIM.transpose() - np.diag(np.diag(current_FIM)) - self.assertTrue(np.all(np.isclose(current_FIM, testing_matrix + np.eye(4)))) + self.assertTrue(np.all(np.isclose(current_FIM, expected_FIM))) def test_ME_opt_greybox_build(self): objective_option = "condition_number" @@ -926,10 +1011,8 @@ def test_ME_opt_greybox_build(self): # Check to see if each component exists all_exist = True - # Check output and value - # FIM Initial will be the prior FIM - # added with the identity matrix. - vals, vecs = np.linalg.eig(testing_matrix + np.eye(4)) + expected_FIM = _._get_FIM() + vals, vecs = np.linalg.eig(expected_FIM) ME_opt_val = np.log(np.abs(np.max(vals) / np.min(vals))) try: @@ -959,13 +1042,14 @@ def test_ME_opt_greybox_build(self): current_FIM[np.triu_indices_from(current_FIM)] = input_values current_FIM += current_FIM.transpose() - np.diag(np.diag(current_FIM)) - self.assertTrue(np.all(np.isclose(current_FIM, testing_matrix + np.eye(4)))) + self.assertTrue(np.all(np.isclose(current_FIM, expected_FIM))) # Testing all the error messages def test_constructor_doe_object_error(self): with self.assertRaisesRegex( ValueError, - "DoE Object must be provided to build external grey box of the FIM.", + "Either ``doe_object`` or both ``parameter_names`` and ``fim_initial`` " + "must be provided to build the FIM grey box.", ): grey_box_object = FIMExternalGreyBox(doe_object=None) @@ -1035,9 +1119,7 @@ def test_evaluate_hessian_outputs_obj_lib_error(self): # Test all versions of solving # using grey box - @unittest.skipIf( - not cyipopt_call_working, "cyipopt is not properly accessing linear solvers" - ) + @unittest.skipIf(not cyipopt_call_working, cyipopt_skip_reason) def test_solve_D_optimality_log_determinant(self): # Two locally optimal design points exist # (time, optimal objective value) @@ -1076,9 +1158,7 @@ def test_solve_D_optimality_log_determinant(self): ) ) - @unittest.skipIf( - not cyipopt_call_working, "cyipopt is not properly accessing linear solvers" - ) + @unittest.skipIf(not cyipopt_call_working, cyipopt_skip_reason) def test_solve_A_optimality_trace_of_inverse(self): # Two locally optimal design points exist # (time, optimal objective value) @@ -1117,9 +1197,7 @@ def test_solve_A_optimality_trace_of_inverse(self): ) ) - @unittest.skipIf( - not cyipopt_call_working, "cyipopt is not properly accessing linear solvers" - ) + @unittest.skipIf(not cyipopt_call_working, cyipopt_skip_reason) @unittest.skipIf(not pandas_available, "pandas is not available") def test_solve_E_optimality_minimum_eigenvalue(self): # Two locally optimal design points exist @@ -1159,9 +1237,7 @@ def test_solve_E_optimality_minimum_eigenvalue(self): ) ) - @unittest.skipIf( - not cyipopt_call_working, "cyipopt is not properly accessing linear solvers" - ) + @unittest.skipIf(not cyipopt_call_working, cyipopt_skip_reason) @unittest.skipIf(not pandas_available, "pandas is not available") def test_solve_ME_optimality_condition_number(self): # Two locally optimal design points exist @@ -1202,5 +1278,610 @@ def test_solve_ME_optimality_condition_number(self): ) +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") +@unittest.skipIf(not numpy_available, "Numpy is not available") +@unittest.skipIf(not scipy_available, "scipy is not available") +@unittest.skipIf(not pandas_available, "pandas is not available") +class TestMultiexperimentBuild(unittest.TestCase): + def test_optimize_experiments_greybox_uses_aggregated_fim(self): + # Check that the multi-experiment greybox block is seeded from the + # aggregated scenario FIM and that the final solve is routed through + # the greybox solver interface. + grey_box_solver = _MockGreyBoxSolver() + doe_obj = _make_multiexperiment_greybox_doe( + objective_option="minimum_eigenvalue", grey_box_solver=grey_box_solver + ) + + doe_obj.optimize_experiments(n_exp=2) + + self.assertEqual(len(grey_box_solver.calls), 1) + self.assertEqual( + doe_obj.results["optimization_solve"]["solver"], grey_box_solver.name + ) + + scenario, total_fim, parameter_names = _get_multiexperiment_scenario_data( + doe_obj + ) + self.assertTrue(hasattr(scenario.obj_cons, "egb_fim_block")) + self.assertFalse(hasattr(scenario.obj_cons, "L")) + + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + if i >= j: + self.assertAlmostEqual( + pyo.value(scenario.obj_cons.egb_fim_block.inputs[(q, p)]), + total_fim[i, j], + places=7, + ) + + self.assertAlmostEqual( + pyo.value(scenario.obj_cons.egb_fim_block.outputs["E-opt"]), + np.min(np.linalg.eigvalsh(total_fim)), + places=7, + ) + + def test_optimize_experiments_greybox_outputs_match_numpy_for_all_supported_objectives( + self, + ): + # Validate the deterministic wiring: for each supported greybox metric, + # the external block output should match direct NumPy on scenario.total_fim. + prior_fim = np.array([[6.0, 0.75], [0.75, 4.0]]) + for objective_option in ( + "determinant", + "trace", + "minimum_eigenvalue", + "condition_number", + ): + with self.subTest(objective=objective_option): + doe_obj = _make_multiexperiment_greybox_doe( + objective_option=objective_option, + prior_FIM=prior_fim.copy(), + grey_box_solver=_MockGreyBoxSolver(name=f"mock-{objective_option}"), + ) + + doe_obj.optimize_experiments(n_exp=2) + + scenario, total_fim, parameter_names = ( + _get_multiexperiment_scenario_data(doe_obj) + ) + egb_block = scenario.obj_cons.egb_fim_block + + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + if i >= j: + self.assertAlmostEqual( + pyo.value(egb_block.inputs[(q, p)]), + total_fim[i, j], + places=7, + ) + + self.assertAlmostEqual( + pyo.value(egb_block.outputs[doe_obj._grey_box_output_name()]), + _expected_multiexperiment_greybox_output( + objective_option, total_fim + ), + places=7, + ) + + def test_optimize_experiments_greybox_prior_fim_is_included_in_inputs_and_output( + self, + ): + # Use a large prior so the external block must see + # total_fim = sum(experiment_fim) + prior_FIM. + prior_fim = np.array([[100.0, 12.0], [12.0, 80.0]]) + doe_obj = _make_multiexperiment_greybox_doe( + objective_option="determinant", + prior_FIM=prior_fim.copy(), + grey_box_solver=_MockGreyBoxSolver(), + ) + + doe_obj.optimize_experiments(n_exp=2) + + scenario, total_fim, parameter_names = _get_multiexperiment_scenario_data( + doe_obj + ) + exp_fim_sum = sum( + ( + np.array(exp_data["fim"]) + for exp_data in doe_obj.results["solution"]["param_scenarios"][0][ + "experiments" + ] + ), + np.zeros_like(total_fim), + ) + egb_block = scenario.obj_cons.egb_fim_block + + self.assertTrue(np.allclose(total_fim, exp_fim_sum + prior_fim, atol=1e-6)) + self.assertFalse(np.allclose(total_fim, exp_fim_sum, atol=1e-6)) + + for i, p in enumerate(parameter_names): + for j, q in enumerate(parameter_names): + if i >= j: + self.assertAlmostEqual( + pyo.value(egb_block.inputs[(q, p)]), total_fim[i, j], places=7 + ) + + output_with_prior = pyo.value(egb_block.outputs["log-D-opt"]) + self.assertAlmostEqual( + output_with_prior, + _expected_multiexperiment_greybox_output("determinant", total_fim), + places=7, + ) + self.assertFalse( + np.isclose( + output_with_prior, + _expected_multiexperiment_greybox_output("determinant", exp_fim_sum), + rtol=1e-6, + atol=1e-6, + ) + ) + + def test_optimize_experiments_greybox_uses_init_solver_for_square_solve_and_grey_box_solver_for_final_solve( + self, + ): + # Initialization should use init_solver; the greybox solver should be + # reserved for the final optimize_experiments NLP solve. + main_solver = _make_ipopt_solver() + init_solver = _make_ipopt_solver() + init_solver.options["max_iter"] = 123 + grey_box_solver = _MockGreyBoxSolver() + + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="minimum_eigenvalue", + use_grey_box_objective=True, + step=1e-2, + solver=main_solver, + grey_box_solver=grey_box_solver, + ) + + init_calls = 0 + call_order = [] + original_init_solve = init_solver.solve + original_grey_box_solve = grey_box_solver.solve + + def _init_solve(*args, **kwargs): + nonlocal init_calls + init_calls += 1 + call_order.append("init") + return original_init_solve(*args, **kwargs) + + def _grey_box_solve(*args, **kwargs): + call_order.append("greybox") + return original_grey_box_solve(*args, **kwargs) + + with ( + patch.object( + main_solver, + "solve", + side_effect=AssertionError( + "Primary solver should not be used in greybox optimize_experiments()." + ), + ), + patch.object(init_solver, "solve", side_effect=_init_solve), + patch.object(grey_box_solver, "solve", side_effect=_grey_box_solve), + ): + doe_obj.optimize_experiments(n_exp=2, init_solver=init_solver) + + self.assertGreaterEqual(init_calls, 1) + self.assertEqual(len(grey_box_solver.calls), 1) + self.assertEqual(call_order[-1], "greybox") + self.assertTrue(all(tag == "init" for tag in call_order[:-1])) + self.assertEqual( + doe_obj.results["initialization"]["solver"], + getattr(init_solver, "name", str(init_solver)), + ) + self.assertEqual( + doe_obj.results["optimization_solve"]["solver"], grey_box_solver.name + ) + + def test_optimize_experiments_greybox_is_reentrant_on_same_object(self): + # Re-running the same greybox DoE object should rebuild a fresh + # external block and reseed it from the current aggregated total FIM. + grey_box_solver = _MockGreyBoxSolver() + doe_obj = _make_multiexperiment_greybox_doe( + objective_option="minimum_eigenvalue", + prior_FIM=np.zeros((2, 2)), + grey_box_solver=grey_box_solver, + ) + + doe_obj.optimize_experiments(n_exp=2) + first_scenario, first_total_fim, _ = _get_multiexperiment_scenario_data(doe_obj) + first_egb_block = first_scenario.obj_cons.egb_fim_block + + self.assertAlmostEqual( + pyo.value(first_egb_block.outputs["E-opt"]), + _expected_multiexperiment_greybox_output( + "minimum_eigenvalue", first_total_fim + ), + places=7, + ) + + doe_obj.prior_FIM = np.array([[20.0, 2.0], [2.0, 15.0]]) + doe_obj.optimize_experiments(n_exp=2) + + second_scenario, second_total_fim, second_parameter_names = ( + _get_multiexperiment_scenario_data(doe_obj) + ) + second_egb_block = second_scenario.obj_cons.egb_fim_block + + self.assertIsNot(first_egb_block, second_egb_block) + self.assertEqual(len(list(doe_obj.model.param_scenario_blocks.keys())), 1) + self.assertEqual(len(grey_box_solver.calls), 2) + self.assertFalse(np.allclose(first_total_fim, second_total_fim, atol=1e-6)) + + for i, p in enumerate(second_parameter_names): + for j, q in enumerate(second_parameter_names): + if i >= j: + self.assertAlmostEqual( + pyo.value(second_egb_block.inputs[(q, p)]), + second_total_fim[i, j], + places=7, + ) + + self.assertAlmostEqual( + pyo.value(second_egb_block.outputs["E-opt"]), + _expected_multiexperiment_greybox_output( + "minimum_eigenvalue", second_total_fim + ), + places=7, + ) + + def test_optimize_experiments_greybox_lhs_initialization_scores_e_opt_and_me_opt( + self, + ): + # This checks the LHS candidate-combination scorer for the greybox-only + # E-opt and ME-opt objectives. The patched oracle maps the single + # Rooney-Biegler design input (hour) to a positive-definite 2x2 FIM so + # the best combination can be computed independently and deterministically. + lhs_n_samples = 4 + lhs_seed = 19 + + for objective_option in ("minimum_eigenvalue", "condition_number"): + with self.subTest(objective=objective_option): + doe_obj = _make_multiexperiment_greybox_doe( + objective_option=objective_option, + prior_FIM=np.zeros((2, 2)), + grey_box_solver=_MockGreyBoxSolver(name=f"mock-{objective_option}"), + ) + + with patch.object( + doe_obj, + "_compute_fim_at_point_no_prior", + side_effect=_spd_hour_fim_oracle, + ): + doe_obj.optimize_experiments( + n_exp=2, + init_method="lhs", + init_n_samples=lhs_n_samples, + init_seed=lhs_seed, + ) + + lhs_init = doe_obj.results["initialization"] + lhs_diag = { + "best_obj": lhs_init["best_initial_objective_value"], + "timed_out": lhs_init["timed_out"], + } + actual_points = lhs_init["selected_initial_designs"] + candidate_points = _generate_lhs_candidate_points( + doe_obj, lhs_n_samples=lhs_n_samples, lhs_seed=lhs_seed + ) + candidate_norm = { + tuple(np.round(point, 8)) for point in candidate_points + } + + self.assertEqual(doe_obj.results["initialization"]["method"], "lhs") + self.assertTrue(np.isfinite(lhs_diag["best_obj"])) + self.assertGreater(lhs_diag["best_obj"], 0.0) + + for point in actual_points: + self.assertIn(tuple(np.round(point, 8)), candidate_norm) + + if doe_obj.objective_option in DesignOfExperiments._MAXIMIZE_OBJECTIVES: + best_obj = -np.inf + is_better = lambda new, best: new > best + else: + best_obj = np.inf + is_better = lambda new, best: new < best + + for combo in itertools.combinations(range(len(candidate_points)), 2): + fim_total = sum( + ( + _spd_hour_fim_oracle(0, candidate_points[idx]) + for idx in combo + ), + np.zeros((2, 2)), + ) + obj_val = doe_obj._evaluate_objective_from_fim(fim_total) + if is_better(obj_val, best_obj): + best_obj = obj_val + + actual_fim_total = sum( + (_spd_hour_fim_oracle(0, point) for point in actual_points), + np.zeros((2, 2)), + ) + actual_obj = doe_obj._evaluate_objective_from_fim(actual_fim_total) + + self.assertAlmostEqual(actual_obj, best_obj, places=12) + self.assertAlmostEqual(lhs_diag["best_obj"], best_obj, places=12) + + def test_optimize_experiments_greybox_initialization_refreshes_inputs_after_square_solve( + self, + ): + # Build-time greybox inputs reflect the template design, but after LHS + # changes the starting point the square-solve refresh should reseed the + # block from the new aggregated FIM before the final solve. + lhs_n_samples = 4 + lhs_seed = 29 + captured = {} + doe_obj = _make_multiexperiment_greybox_doe( + objective_option="minimum_eigenvalue", + prior_FIM=np.zeros((2, 2)), + grey_box_solver=_MockGreyBoxSolver(), + ) + original_initialize = doe_obj._initialize_grey_box_block + + def _capture_initialize(egb_block, fim_np, parameter_names): + captured["before"] = _reconstruct_fim_from_egb_inputs( + egb_block, parameter_names + ) + captured["refreshed"] = np.array(fim_np, copy=True) + return original_initialize(egb_block, fim_np, parameter_names) + + with ( + patch.object( + doe_obj, + "_compute_fim_at_point_no_prior", + side_effect=_diagonal_hour_fim_oracle, + ), + patch.object( + doe_obj, "_initialize_grey_box_block", side_effect=_capture_initialize + ), + ): + doe_obj.optimize_experiments( + n_exp=1, + init_method="lhs", + init_n_samples=lhs_n_samples, + init_seed=lhs_seed, + ) + + scenario, total_fim, parameter_names = _get_multiexperiment_scenario_data( + doe_obj + ) + final_fim = _reconstruct_fim_from_egb_inputs( + scenario.obj_cons.egb_fim_block, parameter_names + ) + + self.assertIn("before", captured) + self.assertIn("refreshed", captured) + self.assertFalse(np.allclose(captured["before"], captured["refreshed"])) + self.assertTrue(np.allclose(final_fim, captured["refreshed"], atol=1e-7)) + self.assertTrue(np.allclose(final_fim, total_fim, atol=1e-7)) + # The Rooney-Biegler template is built with hour=2.0, so this confirms + # LHS moved away from the build-time design and made the refresh check meaningful. + self.assertNotAlmostEqual( + doe_obj.results["initialization"]["selected_initial_designs"][0][0], + 2.0, + places=6, + ) + + def test_optimize_experiments_greybox_tee_flag_reaches_solver(self): + # grey_box_tee is only meaningful if it propagates to the external + # solver interface, so capture the mocked solve() call and verify tee. + grey_box_solver = _MockGreyBoxSolver() + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0)], + objective_option="minimum_eigenvalue", + use_grey_box_objective=True, + step=1e-2, + solver=_make_ipopt_solver(), + grey_box_solver=grey_box_solver, + grey_box_tee=True, + ) + + doe_obj.optimize_experiments(n_exp=2) + + self.assertEqual(len(grey_box_solver.calls), 1) + self.assertTrue(grey_box_solver.calls[0]["tee"]) + + +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") +@unittest.skipIf(not numpy_available, "Numpy is not available") +@unittest.skipIf(not scipy_available, "scipy is not available") +@unittest.skipIf(not cyipopt_available, "'cyipopt' is not available") +@unittest.skipIf( + not cyipopt_call_working, "cyipopt is not properly accessing linear solvers" +) +@unittest.skipIf(not pandas_available, "pandas is not available") +class TestMultiexperimentSolve(unittest.TestCase): + def test_optimize_experiments_determinant_expected_values_greybox(self): + # The scenario objective is order-invariant, so compare the chosen + # experiment hours in sorted order. + exp_list = [ + RooneyBieglerMultiExperiment(hour=1.0, y=8.3), + RooneyBieglerMultiExperiment(hour=2.0, y=10.3), + ] + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="determinant", + step=1e-2, + use_grey_box_objective=True, + grey_box_solver=_make_cyipopt_solver(tol=1e-4), + grey_box_tee=False, + ) + doe.optimize_experiments() + + scenario = doe.results["solution"]["param_scenarios"][0] + got_hours = sorted(exp["design"][0] for exp in scenario["experiments"]) + self.assertStructuredAlmostEqual( + got_hours, sorted([1.9321985035514362, 9.999999685577139]), abstol=1e-3 + ) + self.assertAlmostEqual( + scenario["quality_metrics"]["log10_d_opt"], 6.028152580313302, places=3 + ) + + def test_optimize_experiments_trace_expected_values_greybox(self): + # This is a regression test for the cyipopt-backed multi-experiment + # greybox solve on A-opt: the chosen design and reported objective + # should stay near the known Rooney-Biegler reference solution. + exp_list = [ + RooneyBieglerMultiExperiment(hour=1.0, y=8.3), + RooneyBieglerMultiExperiment(hour=2.0, y=10.3), + ] + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="trace", + step=1e-2, + use_grey_box_objective=True, + grey_box_solver=_make_cyipopt_solver(tol=1e-6), + grey_box_tee=False, + ) + doe.optimize_experiments() + + scenario = doe.results["solution"]["param_scenarios"][0] + got_hours = sorted(exp["design"][0] for exp in scenario["experiments"]) + self.assertStructuredAlmostEqual(got_hours, sorted([1.01, 10.0]), abstol=1e-3) + self.assertAlmostEqual( + scenario["quality_metrics"]["log10_a_opt"], -1.9438, places=3 + ) + + def test_optimize_experiments_min_eig_expected_values_greybox(self): + # This checks the end-to-end greybox E-opt solve against a stable + # reference solution so future greybox wiring changes do not silently + # alter the chosen experiment pair or final metric. + exp_list = [ + RooneyBieglerMultiExperiment(hour=1.0, y=8.3), + RooneyBieglerMultiExperiment(hour=2.0, y=10.3), + ] + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="minimum_eigenvalue", + step=1e-2, + solver=_make_ipopt_solver(), + use_grey_box_objective=True, + grey_box_solver=_make_cyipopt_solver(tol=1e-6), + grey_box_tee=False, + ) + doe.optimize_experiments() + + scenario = doe.results["solution"]["param_scenarios"][0] + got_hours = sorted(exp["design"][0] for exp in scenario["experiments"]) + self.assertStructuredAlmostEqual(got_hours, sorted([1.0, 10.0]), abstol=1e-2) + self.assertAlmostEqual( + scenario["quality_metrics"]["log10_e_opt"], 1.9532, places=2 + ) + + def test_optimize_experiments_me_opt_expected_values_greybox(self): + # ME-opt is greybox-only in optimize_experiments(), so keep a dedicated + # solve regression here to guard the condition-number objective path. + exp_list = [ + RooneyBieglerMultiExperiment(hour=1.0, y=8.3), + RooneyBieglerMultiExperiment(hour=2.0, y=10.3), + ] + + doe = DesignOfExperiments( + experiment=exp_list, + objective_option="condition_number", + step=1e-2, + solver=_make_ipopt_solver(), + use_grey_box_objective=True, + grey_box_solver=_make_cyipopt_solver(tol=1e-6), + grey_box_tee=False, + ) + doe.optimize_experiments() + + scenario = doe.results["solution"]["param_scenarios"][0] + got_hours = sorted(exp["design"][0] for exp in scenario["experiments"]) + self.assertStructuredAlmostEqual(got_hours, sorted([7.13, 10.0]), abstol=1e-2) + self.assertAlmostEqual( + scenario["quality_metrics"]["log10_me_opt"], 1.5289, places=2 + ) + + +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") +@unittest.skipIf(not numpy_available, "Numpy is not available") +@unittest.skipIf(not scipy_available, "scipy is not available") +@unittest.skipIf(not cyipopt_available, "'cyipopt' is not available") +@unittest.skipIf( + not cyipopt_call_working, "cyipopt is not properly accessing linear solvers" +) +@unittest.skipIf(not pandas_available, "pandas is not available") +class TestSingleExperimentSolve(unittest.TestCase): + def test_optimize_experiments_single_experiment_greybox_path_works(self): + # Even with n_exp=1, optimize_experiments() should take the template-mode + # greybox path, solve with the grey_box_solver, and keep the greybox + # block synchronized with the final scenario FIM. + grey_box_solver = _make_cyipopt_solver(tol=1e-6) + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0, y=10.0)], + objective_option="minimum_eigenvalue", + step=1e-2, + solver=_make_ipopt_solver(), + use_grey_box_objective=True, + grey_box_solver=grey_box_solver, + grey_box_tee=False, + ) + + doe_obj.optimize_experiments(n_exp=1) + + scenario, total_fim, _ = _get_multiexperiment_scenario_data(doe_obj) + scenario_results = doe_obj.results["solution"]["param_scenarios"][0] + design_hour = scenario_results["experiments"][0]["design"][0] + + self.assertEqual(doe_obj.results["optimization_solve"]["status"], "ok") + self.assertEqual( + doe_obj.results["problem"]["number_of_experiments_per_scenario"], 1 + ) + self.assertTrue(doe_obj.results["problem"]["used_template_experiment"]) + self.assertEqual(len(scenario_results["experiments"]), 1) + self.assertEqual( + doe_obj.results["optimization_solve"]["solver"], + getattr(grey_box_solver, "name", str(grey_box_solver)), + ) + self.assertGreaterEqual(design_hour, 1.0) + self.assertLessEqual(design_hour, 10.0) + self.assertTrue(np.isfinite(scenario_results["quality_metrics"]["log10_e_opt"])) + self.assertAlmostEqual( + pyo.value(scenario.obj_cons.egb_fim_block.outputs["E-opt"]), + _expected_multiexperiment_greybox_output("minimum_eigenvalue", total_fim), + places=7, + ) + + +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") +@unittest.skipIf(not numpy_available, "Numpy is not available") +@unittest.skipIf(not scipy_available, "scipy is not available") +@unittest.skipIf(not pandas_available, "pandas is not available") +class TestMultiexperimentError(unittest.TestCase): + def test_optimize_experiments_greybox_unsupported_objectives_are_rejected(self): + # These unsupported objectives share the same early-validation path, so + # keep them in one table-driven test and verify none reaches the + # external grey_box_solver interface. + class _UnusedGreyBoxSolver: + def solve(self, model, tee=False): + raise AssertionError("grey_box_solver.solve should not be reached") + + for objective_option in ("pseudo_trace", "zero"): + with self.subTest(objective=objective_option): + doe_obj = DesignOfExperiments( + experiment=[RooneyBieglerMultiExperiment(hour=2.0, y=10.0)], + objective_option=objective_option, + use_grey_box_objective=True, + step=1e-2, + solver=_make_ipopt_solver(), + grey_box_solver=_UnusedGreyBoxSolver(), + ) + + with self.assertRaisesRegex( + ValueError, + "Grey-box objective support in optimize_experiments\\(\\) is only " + "available", + ): + doe_obj.optimize_experiments(n_exp=2) + + if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/doe/tests/test_utils.py b/pyomo/contrib/doe/tests/test_utils.py index a6f6cb38a23..2d22b813941 100644 --- a/pyomo/contrib/doe/tests/test_utils.py +++ b/pyomo/contrib/doe/tests/test_utils.py @@ -6,10 +6,18 @@ # Solutions of Sandia, LLC, the U.S. Government retains certain rights in this # software. This software is distributed under the 3-clause BSD License. # ____________________________________________________________________________________ -from pyomo.common.dependencies import numpy as np, numpy_available +from pyomo.common.dependencies import ( + numpy as np, + numpy_available, + pandas as pd, + pandas_available, + scipy_available, +) import pyomo.common.unittest as unittest +import pyomo.environ as pyo from pyomo.contrib.doe.utils import ( + ExperimentGradients, check_FIM, compute_FIM_metrics, get_FIM_metrics, @@ -17,8 +25,19 @@ _SMALL_TOLERANCE_SYMMETRY, _SMALL_TOLERANCE_IMG, ) +if not (numpy_available and scipy_available): + raise unittest.SkipTest("Pyomo.DoE needs scipy and numpy to run tests") + +from pyomo.contrib.doe.examples.polynomial import PolynomialExperiment +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) +from pyomo.opt import SolverFactory + +ipopt_available = SolverFactory("ipopt").available() +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") @unittest.skipIf(not numpy_available, "Numpy is not available") class TestUtilsFIM(unittest.TestCase): """Test the check_FIM() from utils.py.""" @@ -153,6 +172,196 @@ def test_get_FIM_metrics(self): fim_metrics["log10(Modified E-Optimality)"], expected['ME_opt'] ) +@unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") +@unittest.skipIf(not numpy_available, "Numpy is not available") +class TestExperimentGradients(unittest.TestCase): + """Validate symbolic and automatic differentiation helpers.""" + + def _assert_symbolic_and_automatic_jacobians_agree( + self, model, atol=1e-8, rtol=1e-8 + ): + """Check that symbolic and automatic Jacobian tables agree entry by entry.""" + experiment_gradients = ExperimentGradients(model, symbolic=True, automatic=True) + + self.assertEqual( + set(experiment_gradients.jac_dict_sd), set(experiment_gradients.jac_dict_ad) + ) + for key in experiment_gradients.jac_dict_sd: + self.assertTrue( + np.isclose( + pyo.value(experiment_gradients.jac_dict_sd[key]), + pyo.value(experiment_gradients.jac_dict_ad[key]), + atol=atol, + rtol=rtol, + ), + msg=f"Mismatch at Jacobian entry {key}", + ) + return experiment_gradients + + def _get_rooney_biegler_experiment( + self, hour=5.0, y=15.6, asymptote=15.0, rate_constant=0.5, measure_error=0.1 + ): + """Build a Rooney-Biegler experiment for gradient validation.""" + data = pd.DataFrame(data=[[hour, y]], columns=["hour", "y"]) + return RooneyBieglerExperiment( + data=data.iloc[0], + theta={"asymptote": asymptote, "rate_constant": rate_constant}, + measure_error=measure_error, + ) + + def _get_expected_polynomial_gradient(self): + """Return the exact gradient of the polynomial output at the test point.""" + return np.array([[2.0, 3.0, 6.0, 1.0]]) + + def _get_expected_polynomial_fim_with_prior(self): + """Return a positive-definite polynomial FIM used for metric regression.""" + gradient = self._get_expected_polynomial_gradient().ravel() + return np.outer(gradient, gradient) + np.eye(4) + + def _evaluate_polynomial_output(self, a, b, c, d, x1=2.0, x2=3.0): + """Evaluate the scalar polynomial model at the test point.""" + return a * x1 + b * x2 + c * x1 * x2 + d + + def test_polynomial_gradients_match_expected(self): + """Check polynomial output sensitivities against analytic values.""" + experiment = PolynomialExperiment() + model = experiment.get_labeled_model() + + experiment_gradients = ExperimentGradients(model, symbolic=True, automatic=True) + jacobian = ( + experiment_gradients.compute_gradient_outputs_wrt_unknown_parameters() + ) + + expected = self._get_expected_polynomial_gradient() + + self.assertEqual(jacobian.shape, expected.shape) + self.assertTrue(np.allclose(jacobian, expected)) + + def test_polynomial_symbolic_and_automatic_jacobians_agree(self): + """Ensure symbolic and automatic Jacobian entries agree exactly.""" + experiment = PolynomialExperiment() + model = experiment.get_labeled_model() + + self._assert_symbolic_and_automatic_jacobians_agree(model) + + def test_polynomial_symbolic_matches_manual_central_difference(self): + """Check symbolic sensitivities against a manual central-difference estimate.""" + experiment = PolynomialExperiment() + model = experiment.get_labeled_model() + experiment_gradients = ExperimentGradients(model, symbolic=True, automatic=True) + + symbolic = ( + experiment_gradients.compute_gradient_outputs_wrt_unknown_parameters() + .ravel() + .astype(float) + ) + base_values = {"a": 2.0, "b": -1.0, "c": 0.5, "d": -1.0} + step = 1e-6 + finite_difference = [] + for parameter in ("a", "b", "c", "d"): + forward_values = dict(base_values) + backward_values = dict(base_values) + forward_values[parameter] += step + backward_values[parameter] -= step + forward = self._evaluate_polynomial_output(**forward_values) + backward = self._evaluate_polynomial_output(**backward_values) + finite_difference.append((forward - backward) / (2 * step)) + + self.assertTrue(np.allclose(symbolic, finite_difference, atol=1e-7, rtol=1e-7)) + + def test_polynomial_automatic_only_still_sets_both_jacobians(self): + """Check that both Jacobian maps are prepared in the unified setup path.""" + experiment = PolynomialExperiment() + model = experiment.get_labeled_model() + + experiment_gradients = ExperimentGradients( + model, symbolic=False, automatic=True + ) + + jacobian =( + experiment_gradients.compute_gradient_outputs_wrt_unknown_parameters() + ) + expected = self._get_expected_polynomial_gradient() + + self.assertIsNotNone(experiment_gradients.jac_dict_sd) + self.assertIsNotNone(experiment_gradients.jac_dict_ad) + self.assertEqual(jacobian.shape, expected.shape) + self.assertTrue(np.allclose(jacobian,expected)) + + + def test_polynomial_symbolic_only_still_sets_both_jacobians(self): + """Check that symbolic-only requests still initialize both Jacobian maps.""" + experiment = PolynomialExperiment() + model = experiment.get_labeled_model() + + experiment_gradients = ExperimentGradients( + model, symbolic=True, automatic=False + ) + + jacobian =( + experiment_gradients.compute_gradient_outputs_wrt_unknown_parameters() + ) + expected = self._get_expected_polynomial_gradient() + + self.assertIsNotNone(experiment_gradients.jac_dict_sd) + self.assertIsNotNone(experiment_gradients.jac_dict_ad) + + @unittest.skipIf(not pandas_available, "pandas is not available") + def test_rooney_biegler_symbolic_and_automatic_jacobians_agree(self): + """Check Rooney-Biegler Jacobians from symbolic and automatic differentiation.""" + experiment = self._get_rooney_biegler_experiment() + model = experiment.get_labeled_model() + + self._assert_symbolic_and_automatic_jacobians_agree(model) + + @unittest.skipIf(not pandas_available, "pandas is not available") + def test_rooney_biegler_gradients_match_closed_form(self): + """Check Rooney-Biegler sensitivities against the closed-form derivatives.""" + hour = 7.0 + asymptote = 14.0 + rate_constant = 0.4 + experiment = self._get_rooney_biegler_experiment( + hour=hour, y=19.8, asymptote=asymptote, rate_constant=rate_constant + ) + model = experiment.get_labeled_model() + experiment_gradients = self._assert_symbolic_and_automatic_jacobians_agree( + model + ) + + jacobian = ( + experiment_gradients.compute_gradient_outputs_wrt_unknown_parameters() + ) + expected = np.array( + [ + [ + 1.0 - np.exp(-rate_constant * hour), + asymptote * hour * np.exp(-rate_constant * hour), + ] + ] + ) + + self.assertEqual(jacobian.shape, expected.shape) + self.assertTrue(np.allclose(jacobian, expected, atol=1e-7, rtol=1e-7)) + + + + @unittest.skipIf(not scipy_available, "scipy is not available") + @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available") + def test_rooney_biegler_symbolic_and_automatic_jacobians_agree_at_perturbed_point(self): + """Check Rooney-Biegler Jacobian agreement at a perturbed operating point.""" + experiment = self._get_rooney_biegler_experiment(hour=7.0, y=19.8, asymptote=14.0, rate_constant=0.4) + model = experiment.get_labeled_model() + + experiment_gradients = self._assert_symbolic_and_automatic_jacobians_agree( + model, atol=1e-6, rtol=1e-6 + ) + + + self.assertGreater(len(experiment_gradients.jac_dict_sd), 0) + self.assertEqual( + len(experiment_gradients.jac_dict_sd), len(experiment_gradients.jac_dict_ad) + ) + if __name__ == "__main__": unittest.main() diff --git a/pyomo/contrib/doe/utils.py b/pyomo/contrib/doe/utils.py index c85fbbe4c0b..22b63719e4b 100644 --- a/pyomo/contrib/doe/utils.py +++ b/pyomo/contrib/doe/utils.py @@ -27,9 +27,12 @@ import pyomo.environ as pyo from pyomo.common.dependencies import numpy as np, numpy_available +from pyomo.common.collections import ComponentSet from pyomo.core.base.param import ParamData from pyomo.core.base.var import VarData +from pyomo.core.expr.calculus.diff_with_pyomo import reverse_sd, reverse_ad +from pyomo.core.expr.visitor import identify_variables import logging @@ -86,6 +89,9 @@ def rescale_FIM(FIM, param_vals): raise ValueError( "param_vals should be a list or numpy array of dimensions: 1 by `n_params`" ) + # Form the matrix with entries scaling_mat[i, j] = 1 / (theta_i theta_j), where theta_i + # and theta_j are the i-th and j-th parameter values. The scaled FIM is then + # computed elementwise as scaled_FIM[i, j] = FIM[i, j] / (theta_i theta_j). scaling_mat = (1 / param_vals).transpose().dot((1 / param_vals)) scaled_FIM = np.multiply(FIM, scaling_mat) return scaled_FIM @@ -162,18 +168,29 @@ def compute_FIM_metrics(FIM): # Check whether the FIM is square, positive definite, and symmetric check_FIM(FIM) + # D-optimality uses det(FIM); larger determinant means a smaller parameter + # confidence ellipsoid, i.e., tighter joint parameter uncertainty. - # Compute FIM metrics det_FIM = np.linalg.det(FIM) D_opt = np.log10(det_FIM) # Trace of FIM is the pseudo A-optimality, not the proper definition of A-optimality, # The trace of covariance is the proper definition of A-optimality + # trace(FIM) gives a convenient proxy for total information, while + # trace(FIM^{-1}) gives the standard A-optimality metric based on + # total parameter variance. + # A-optimality geometrically minimizes the average + # squared semi-axis length of the parameter confidence ellipsoid. + trace_FIM = np.trace(FIM) pseudo_A_opt = np.log10(trace_FIM) trace_cov = np.trace(np.linalg.pinv(FIM)) A_opt = np.log10(trace_cov) + # E-optimality uses the smallest eigenvalue of the FIM, so it targets the + # worst-identified parameter direction by minimizing the longest axis of the + # confidence ellipsoid. + E_vals, E_vecs = np.linalg.eig(FIM) E_ind = np.argmin(E_vals.real) # index of smallest eigenvalue @@ -190,6 +207,10 @@ def compute_FIM_metrics(FIM): else: E_opt = np.log10(E_vals.real[E_ind]) + # Modified E-optimality is based on the FIM condition number and penalizes + # confidence ellipsoids that are highly elongated in one direction. + + ME_opt = np.log10(np.linalg.cond(FIM)) return ( @@ -266,3 +287,224 @@ def get_FIM_metrics(FIM): "log10(E-Optimality)": E_opt, "log10(Modified E-Optimality)": ME_opt, } + + +class ExperimentGradients: + """Utilities for differentiating labeled experiment models. + + This helper implements the symbolic sensitivity path used in Pyomo.DoE. + Instead of approximating sensitivities by finite-difference perturbations + of the unknown parameters theta, it differentiates the + model F(x, u, theta) = 0 with respect to theta, with the design variables + u fixed, and solves the resulting auxiliary sensitivity system + + dF/dx * dx/dtheta + dF/dtheta = 0 + + to obtain the local output sensitivities dy/dtheta needed for Fisher + information matrix calculations. + """ + + def __init__(self, experiment_model, symbolic=True, automatic=True, verbose=False): + self.model = experiment_model + self.verbose = verbose + + self._analyze_experiment_model() + + self.jac_dict_sd = None + self.jac_dict_ad = None + self.jac_measurements_wrt_param = None + + self._requested_symbolic = symbolic + self._requested_automatic = automatic + + if symbolic or automatic: + self._setup_differentiation() + + def _analyze_experiment_model(self): + """Build index mappings for constraints, variables, parameters, and outputs. + + This inspects the labeled experiment model and records the ordered + constraint and variable lists used later to assemble Jacobian blocks for + sensitivity calculations for F(x, u, theta) = 0. It also tracks which + indexed quantities correspond to unknown parameters theta and measured + outputs y. + """ + model = self.model + + # Fix the design variables u and unknown parameters theta so the + # remaining active equations define the model F(x, u, theta) = 0 + # used for sensitivity calculations. + for v in model.experiment_inputs.keys(): + v.fix() + for v in model.unknown_parameters.keys(): + v.fix() + + param_set = ComponentSet(model.unknown_parameters.keys()) + output_set = ComponentSet(model.experiment_outputs.keys()) + con_set = ComponentSet() + var_set = ComponentSet() + + for c in model.component_data_objects( + pyo.Constraint, descend_into=True, active=True + ): + con_set.add(c) + for v in identify_variables(c.body, include_fixed=False): + var_set.add(v) + + # The parameters theta may not appear in identify_variables(..., + # include_fixed=False) after being fixed, but they still need indexed + # columns in the full Jacobian. + for p in model.unknown_parameters.keys(): + if p not in var_set: + var_set.add(p) + + measurement_mapping = pyo.Suffix(direction=pyo.Suffix.LOCAL) + parameter_mapping = pyo.Suffix(direction=pyo.Suffix.LOCAL) + measurement_error_included = pyo.Suffix(direction=pyo.Suffix.LOCAL) + + con_list = list(con_set) + var_list = list(var_set) + + param_index = [] + model_var_index = [] + measurement_index = [] + + # Partition the indexed quantities into parameter columns (theta), + # model-variable columns (x), and measured-output rows (y) for later + # Jacobian slicing. + for i, v in enumerate(var_list): + if v in param_set: + param_index.append(i) + parameter_mapping[v] = i + else: + model_var_index.append(i) + if v in output_set: + measurement_index.append(i) + measurement_error_included[v] = model.measurement_error[v] + measurement_mapping[v] = i + + for o in model.experiment_outputs.keys(): + if o not in var_set: + measurement_mapping[o] = None + + self.con_list = con_list + self.var_list = var_list + self.param_index = param_index + self.model_var_index = model_var_index + self.measurement_index = measurement_index + self.measurement_error_included = measurement_error_included + self.num_measurements = len(output_set) + self.num_params = len(param_set) + self.num_constraints = len(con_set) + self.num_vars = len(var_set) + self.var_set = var_set + self.measurement_mapping = measurement_mapping + self.parameter_mapping = parameter_mapping + + def _setup_differentiation(self): + """Build symbolic and automatic Jacobian maps in a single pass.""" + if not self._requested_symbolic and not self._requested_automatic: + raise ValueError("At least one differentiation method must be selected.") + + jac_dict_sd = {} + jac_dict_ad = {} + + for i, c in enumerate(self.con_list): + if not c.equality: + raise ValueError( + "ExperimentGradients currently requires equality constraints." + ) + + # For each equation F_i(x, u, theta) = 0, compute the partial derivatives with + # respect to the indexed variables and parameters. These derivatives form the + # Jacobian rows used to assemble the blocks dF/dx and dF/dtheta in the + # sensitivity system dF/dx * dx/dtheta + dF/dtheta = 0. + + der_map_sd = reverse_sd(c.body) + der_map_ad = reverse_ad(c.body) + + for j, v in enumerate(self.var_list): + # If a variable or parameter does not appear in F_i, its partial derivative in + # that equation is zero, so absent derivative entries are filled in with 0. + jac_dict_sd[(i, j)] = der_map_sd.get(v, 0) + jac_dict_ad[(i, j)] = der_map_ad.get(v, 0) + + self.jac_dict_sd = jac_dict_sd + self.jac_dict_ad = jac_dict_ad + + def compute_gradient_outputs_wrt_unknown_parameters(self): + """Compute the output sensitivity matrix with respect to theta. + + This differentiates the model F(x, u, theta) = 0 with u fixed, solves + for dx/dtheta, and then extracts the measured-output rows to return + dy/dtheta. + """ + if self.jac_dict_ad is None: + self._setup_differentiation() + + # Assemble dF/dtheta, the Jacobian block of the model equations with + # respect to the unknown parameters theta. + jac_con_wrt_param = np.zeros((self.num_constraints, self.num_params)) + for i in range(self.num_constraints): + for j, p in enumerate(self.param_index): + jac_con_wrt_param[i, j] = self.jac_dict_ad[(i, p)] + + # Assemble dF/dx, the Jacobian block of the model equations with + # respect to the model variables x. + jac_con_wrt_vars = np.zeros((self.num_constraints, len(self.model_var_index))) + for i in range(self.num_constraints): + for j, v in enumerate(self.model_var_index): + jac_con_wrt_vars[i, j] = self.jac_dict_ad[(i, v)] + + # With the design variables u fixed, differentiate F(x, u, theta) = 0 + # to obtain dF/dx * dx/dtheta + dF/dtheta = 0, then solve for + # dx/dtheta = -(dF/dx)^{-1}(dF/dtheta). + jac_vars_wrt_param = np.linalg.solve(jac_con_wrt_vars, -jac_con_wrt_param) + + # Extract the rows of dx/dtheta corresponding to the measured outputs y + # to form the sensitivity matrix dy/dtheta used in the FIM. + jac_measurements_wrt_param = np.zeros((self.num_measurements, self.num_params)) + for ind, m in enumerate(self.model.experiment_outputs.keys()): + i = self.measurement_mapping[m] + if i is None: + jac_measurements_wrt_param[ind, :] = 0.0 + else: + jac_measurements_wrt_param[ind, :] = jac_vars_wrt_param[i, :] + + self.jac_measurements_wrt_param = jac_measurements_wrt_param + return jac_measurements_wrt_param + + def construct_sensitivity_constraints(self, model=None): + """Add symbolic sensitivity variables and constraints to a Pyomo model. + + The added constraints encode the differentiated model equations + dF/dx * dx/dtheta + dF/dtheta = 0, where F(x, u, theta) = 0 is the + model, x is the vector of model variables, u is the + vector of design variables, and theta is the vector of unknown + parameters. This makes the local sensitivities dx/dtheta explicit + inside the optimization model. + """ + if self.jac_dict_sd is None: + self._setup_differentiation() + + if model is None: + model = self.model + + model.param_index = pyo.Set(initialize=self.param_index) + model.constraint_index = pyo.Set(initialize=range(len(self.con_list))) + model.var_index = pyo.Set(initialize=self.model_var_index) + # Introduce Pyomo variables representing dx/dtheta so the local + # sensitivity system can be written explicitly inside the optimization + # model. + model.jac_variables_wrt_param = pyo.Var( + model.var_index, model.param_index, initialize=0 + ) + + @model.Constraint(model.constraint_index, model.param_index) + def jacobian_constraint(model, i, j): + # Enforce dF/dx * dx/dtheta + dF/dtheta = 0 for each model equation + # and parameter. + return self.jac_dict_sd[(i, j)] == -sum( + model.jac_variables_wrt_param[k, j] * self.jac_dict_sd[(i, k)] + for k in model.var_index + ) diff --git a/pyomo/contrib/parmest/examples/reaction_kinetics/simple_reaction_parmest_example.py b/pyomo/contrib/parmest/examples/reaction_kinetics/simple_reaction_parmest_example.py index 72eeb606fc2..301c2bebb30 100644 --- a/pyomo/contrib/parmest/examples/reaction_kinetics/simple_reaction_parmest_example.py +++ b/pyomo/contrib/parmest/examples/reaction_kinetics/simple_reaction_parmest_example.py @@ -52,23 +52,6 @@ def simple_reaction_model(data): # fix all of the regressed parameters model.k.fix() - # =================================================================== - # Stage-specific cost computations - def ComputeFirstStageCost_rule(model): - return 0 - - model.FirstStageCost = Expression(rule=ComputeFirstStageCost_rule) - - def AllMeasurements(m): - return (float(data['y']) - m.y) ** 2 - - model.SecondStageCost = Expression(rule=AllMeasurements) - - def total_cost_rule(m): - return m.FirstStageCost + m.SecondStageCost - - model.Total_Cost_Objective = Objective(rule=total_cost_rule, sense=minimize) - return model @@ -90,6 +73,8 @@ def label_model(self): m.experiment_outputs.update( [(m.x1, self.data['x1']), (m.x2, self.data['x2']), (m.y, self.data['y'])] ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None), (m.x1, None), (m.x2, None)]) return m @@ -161,7 +146,7 @@ def main(): # Only estimate the parameter k[1]. The parameter k[2] will remain fixed # at its initial value - pest = parmest.Estimator(exp_list) + pest = parmest.Estimator(exp_list, obj_function="SSE") obj, theta = pest.theta_est() print(obj) print(theta) @@ -174,9 +159,11 @@ def main(): # ======================================================================= # Estimate both k1 and k2 and compute the covariance matrix - pest = parmest.Estimator(exp_list) - n = 15 # total number of data points used in the objective (y in 15 scenarios) - obj, theta, cov = pest.theta_est(calc_cov=True, cov_n=n) + pest = parmest.Estimator(exp_list, obj_function="SSE") + # Calculate the objective value and estimated parameters + obj, theta = pest.theta_est() + # Compute the covariance matrix using the reduced Hessian method + cov = pest.cov_est(method="reduced_hessian") print(obj) print(theta) print(cov) diff --git a/pyomo/contrib/parmest/examples/reactor_design/multistart_example.py b/pyomo/contrib/parmest/examples/reactor_design/multistart_example.py new file mode 100644 index 00000000000..cfb9eea9798 --- /dev/null +++ b/pyomo/contrib/parmest/examples/reactor_design/multistart_example.py @@ -0,0 +1,56 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + +from pyomo.common.dependencies import numpy as np, pandas as pd +from itertools import product +from os.path import join, abspath, dirname +import pyomo.contrib.parmest.parmest as parmest +from pyomo.contrib.parmest.examples.reactor_design.reactor_design import ( + ReactorDesignExperiment, +) + + +def main(): + + # Read in data + file_dirname = dirname(abspath(str(__file__))) + file_name = abspath(join(file_dirname, "reactor_data.csv")) + data = pd.read_csv(file_name) + + # Create an experiment list + exp_list = [ReactorDesignExperiment(data, i) for i in range(data.shape[0])] + + # Solver options belong here (Ipopt options shown as example) + solver_options = {"max_iter": 1000, "tol": 1e-6} + + pest = parmest.Estimator( + exp_list, obj_function="SSE", solver_options=solver_options + ) + + # Single-start estimation + obj, theta = pest.theta_est() + print("Single-start objective:", obj) + print("Single-start theta:\n", theta) + + # Multistart estimation + results_df, best_theta, best_obj = pest.theta_est_multistart( + n_restarts=10, + multistart_sampling_method="uniform_random", + seed=42, + save_results=False, # True if you want CSV via file_name= + ) + + print("\nMultistart best objective:", best_obj) + print("Multistart best theta:", best_theta) + print("\nAll multistart results:") + print(results_df) + + +if __name__ == "__main__": + main() diff --git a/pyomo/contrib/parmest/examples/reactor_design/parameter_estimation_example.py b/pyomo/contrib/parmest/examples/reactor_design/parameter_estimation_example.py index 630802ddba9..a5a644a4c7b 100644 --- a/pyomo/contrib/parmest/examples/reactor_design/parameter_estimation_example.py +++ b/pyomo/contrib/parmest/examples/reactor_design/parameter_estimation_example.py @@ -33,11 +33,17 @@ def main(): pest = parmest.Estimator(exp_list, obj_function='SSE') - # Parameter estimation with covariance - obj, theta, cov = pest.theta_est(calc_cov=True, cov_n=17) - print(obj) + # Parameter estimation + obj, theta = pest.theta_est() + print("Least squares objective value:", obj) + print("Estimated parameters (theta):\n") print(theta) + # Compute the covariance matrix at the estimated parameter + cov = pest.cov_est() + print("Covariance matrix:\n") + print(cov) + if __name__ == "__main__": main() diff --git a/pyomo/contrib/parmest/examples/reactor_design/reactor_design.py b/pyomo/contrib/parmest/examples/reactor_design/reactor_design.py index 1b3360a2d93..6a065d20a63 100644 --- a/pyomo/contrib/parmest/examples/reactor_design/reactor_design.py +++ b/pyomo/contrib/parmest/examples/reactor_design/reactor_design.py @@ -22,15 +22,15 @@ def reactor_design_model(): # Create the concrete model model = pyo.ConcreteModel() - # Rate constants - model.k1 = pyo.Param( - initialize=5.0 / 6.0, within=pyo.PositiveReals, mutable=True + # Rate constants, make unknown parameters variables + model.k1 = pyo.Var( + initialize=5.0 / 6.0, within=pyo.PositiveReals, bounds=(0.1, 10.0) ) # min^-1 - model.k2 = pyo.Param( - initialize=5.0 / 3.0, within=pyo.PositiveReals, mutable=True + model.k2 = pyo.Var( + initialize=5.0 / 3.0, within=pyo.PositiveReals, bounds=(0.1, 10.0) ) # min^-1 - model.k3 = pyo.Param( - initialize=1.0 / 6000.0, within=pyo.PositiveReals, mutable=True + model.k3 = pyo.Var( + initialize=1.0 / 6000.0, within=pyo.PositiveReals, bounds=(1e-5, 1e-3) ) # m^3/(gmol min) # Inlet concentration of A, gmol/m^3 @@ -119,6 +119,11 @@ def label_model(self): (k, pyo.ComponentUID(k)) for k in [m.k1, m.k2, m.k3] ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update( + [(m.ca, None), (m.cb, None), (m.cc, None), (m.cd, None)] + ) + return m def get_labeled_model(self): diff --git a/pyomo/contrib/parmest/examples/rooney_biegler/multistart_example.py b/pyomo/contrib/parmest/examples/rooney_biegler/multistart_example.py new file mode 100644 index 00000000000..7ff8f55ad4b --- /dev/null +++ b/pyomo/contrib/parmest/examples/rooney_biegler/multistart_example.py @@ -0,0 +1,62 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + +from pyomo.common.dependencies import numpy as np, pandas as pd +from itertools import product +import pyomo.contrib.parmest.parmest as parmest +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) + + +def main(): + + # Data + data = pd.DataFrame( + data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0], [5, 15.6], [7, 19.8]], + columns=['hour', 'y'], + ) + + # Create an experiment list + exp_list = [] + for i in range(data.shape[0]): + exp_list.append(RooneyBieglerExperiment(data.loc[i, :])) + + # View one model + # exp0_model = exp_list[0].get_labeled_model() + # exp0_model.pprint() + + # Solver options belong here (Ipopt options shown as example) + solver_options = {"max_iter": 1000, "tol": 1e-6} + + pest = parmest.Estimator( + exp_list, obj_function="SSE", solver_options=solver_options + ) + + # Single-start estimation + obj, theta = pest.theta_est() + print("Single-start objective:", obj) + print("Single-start theta:\n", theta) + + # Multistart estimation + results_df, best_theta, best_obj = pest.theta_est_multistart( + n_restarts=10, + multistart_sampling_method="uniform_random", + seed=42, + save_results=False, # True if you want CSV via file_name= + ) + + print("\nMultistart best objective:", best_obj) + print("Multistart best theta:", best_theta) + print("\nAll multistart results:") + print(results_df) + + +if __name__ == "__main__": + main() diff --git a/pyomo/contrib/parmest/examples/rooney_biegler/profile_likelihood_example.py b/pyomo/contrib/parmest/examples/rooney_biegler/profile_likelihood_example.py new file mode 100644 index 00000000000..8d3bb2b22ef --- /dev/null +++ b/pyomo/contrib/parmest/examples/rooney_biegler/profile_likelihood_example.py @@ -0,0 +1,74 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + +from pyomo.common.dependencies import pandas as pd +import pyomo.contrib.parmest.parmest as parmest +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) + + +def main(): + # Data + data = pd.DataFrame( + data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0], [5, 15.6], [7, 19.8]], + columns=["hour", "y"], + ) + + # Build experiment list + exp_list = [RooneyBieglerExperiment(data.loc[i, :]) for i in range(data.shape[0])] + + # Create estimator + pest = parmest.Estimator(exp_list, obj_function="SSE") + + # Compute profile likelihood for both unknown parameters. + # Use a small grid for quick terminal runs. + profile_results = pest.profile_likelihood( + profiled_theta=["asymptote", "rate_constant"], + n_grid=9, + solver="ef_ipopt", + warmstart="neighbor", + # Demonstrate baseline from multistart integration: + use_multistart_for_baseline=True, + baseline_multistart_kwargs={ + "n_restarts": 5, + "multistart_sampling_method": "uniform_random", + "seed": 7, + }, + ) + + # Display a compact summary table + profiles = profile_results["profiles"] + print("\nBaseline:") + print(profile_results["baseline"]) + print("\nProfile results (first 12 rows):") + print( + profiles[ + [ + "profiled_theta", + "theta_value", + "obj", + "delta_obj", + "lr_stat", + "status", + "success", + ] + ].head(12) + ) + + # Plot profile curves to file for terminal/non-GUI usage + out_file = "rooney_biegler_profile_likelihood.png" + parmest.graphics.profile_likelihood_plot( + profile_results, alpha=0.95, filename=out_file + ) + print(f"\nSaved plot to: {out_file}") + + +if __name__ == "__main__": + main() diff --git a/pyomo/contrib/parmest/examples/rooney_biegler/regularization_example.py b/pyomo/contrib/parmest/examples/rooney_biegler/regularization_example.py new file mode 100644 index 00000000000..acdd9efa2ab --- /dev/null +++ b/pyomo/contrib/parmest/examples/rooney_biegler/regularization_example.py @@ -0,0 +1,114 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + +from pyomo.common.dependencies import pandas as pd +import pyomo.contrib.parmest.parmest as parmest +from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import ( + RooneyBieglerExperiment, +) + + +def main(): + """ + Compare L2 and smooth-L1 regularization on the Rooney-Biegler example. + + Notes + ----- + The model response saturates for large positive ``rate_constant`` values: + ``y = asymptote * (1 - exp(-rate_constant * hour))``. + If ``rate_constant`` is both unpenalized and unbounded, the objective can be + nearly flat in that direction, which can lead to extreme fitted values. + To keep the smooth-L1 fit numerically stable and interpretable, this example: + 1) includes a nonzero L1 weight on ``rate_constant`` via ``prior_FIM_l1``, and + 2) applies finite bounds ``rate_constant in [0, 5]`` on each experiment model. + """ + + # Rooney & Biegler Reference Values + # a = 19.14, b = 0.53 + theta_ref = pd.Series({'asymptote': 20.0, 'rate_constant': 0.8}) + + # L2 setup: create a 'Stiff' Prior for 'asymptote' but leave 'rate_constant' flexible. + prior_FIM_l2 = pd.DataFrame( + [[1000.0, 0.0], [0.0, 0.0]], + index=['asymptote', 'rate_constant'], + columns=['asymptote', 'rate_constant'], + ) + # L1 setup: penalize both parameters to avoid an unregularized flat direction. + prior_FIM_l1 = pd.DataFrame( + [[1000.0, 0.0], [0.0, 1.0]], + index=['asymptote', 'rate_constant'], + columns=['asymptote', 'rate_constant'], + ) + + # Data + data = pd.DataFrame( + data=[[1, 8.3], [2, 10.3], [3, 19.0], [4, 16.0], [5, 15.6], [7, 19.8]], + columns=['hour', 'y'], + ) + + # Create an experiment list + exp_list = [] + for i in range(data.shape[0]): + exp = RooneyBieglerExperiment(data.loc[i, :]) + # Example-scoped stabilization: keep rate_constant in a practical range. + m = exp.get_labeled_model() + m.rate_constant.setlb(0.0) + m.rate_constant.setub(5.0) + exp_list.append(exp) + + # Create an instance of the parmest estimator (L2) + pest_l2 = parmest.Estimator( + exp_list, + obj_function="SSE", + regularization='L2', + prior_FIM=prior_FIM_l2, + theta_ref=theta_ref, + ) + + # Parameter estimation and covariance for L2 + obj_l2, theta_l2 = pest_l2.theta_est() + cov_l2 = pest_l2.cov_est() + + # L1 smooth regularization uses sqrt((theta - theta_ref)^2 + epsilon) + pest_l1 = parmest.Estimator( + exp_list, + obj_function="SSE", + regularization='L1', + prior_FIM=prior_FIM_l1, + theta_ref=theta_ref, + regularization_weight=1.0, + regularization_epsilon=1e-6, + ) + obj_l1, theta_l1 = pest_l1.theta_est() + cov_l1 = pest_l1.cov_est() + + if parmest.graphics.seaborn_available: + parmest.graphics.pairwise_plot( + (theta_l2, cov_l2, 100), + theta_star=theta_l2, + alpha=0.8, + distributions=['MVN'], + title='L2 regularized theta estimates within 80% confidence region', + ) + + # Assert statements compare parameter estimation (theta) to an expected value + # relative_error = abs(theta['asymptote'] - 19.1426) / 19.1426 + # assert relative_error < 0.01 + # relative_error = abs(theta['rate_constant'] - 0.5311) / 0.5311 + # assert relative_error < 0.01 + + return {"L2": (obj_l2, theta_l2, cov_l2), "L1": (obj_l1, theta_l1, cov_l1)} + + +if __name__ == "__main__": + results = main() + for reg_name, (obj, theta, cov) in results.items(): + print(f"{reg_name} estimated parameters (theta):", theta) + print(f"{reg_name} objective function value at theta:", obj) + print(f"{reg_name} covariance of parameter estimates:", cov) diff --git a/pyomo/contrib/parmest/examples/rooney_biegler/rooney_biegler.py b/pyomo/contrib/parmest/examples/rooney_biegler/rooney_biegler.py index bdb494bca03..4ed4e8fc947 100644 --- a/pyomo/contrib/parmest/examples/rooney_biegler/rooney_biegler.py +++ b/pyomo/contrib/parmest/examples/rooney_biegler/rooney_biegler.py @@ -43,8 +43,8 @@ def rooney_biegler_model(data, theta=None): if theta is None: theta = {'asymptote': 15, 'rate_constant': 0.5} - model.asymptote = pyo.Var(initialize=theta['asymptote']) - model.rate_constant = pyo.Var(initialize=theta['rate_constant']) + model.asymptote = pyo.Var(initialize=theta['asymptote'], bounds=(0.1, 100)) + model.rate_constant = pyo.Var(initialize=theta['rate_constant'], bounds=(0, 10)) # Fix the unknown parameters model.asymptote.fix() diff --git a/pyomo/contrib/parmest/graphics.py b/pyomo/contrib/parmest/graphics.py index bed9f9eb824..9361c737b61 100644 --- a/pyomo/contrib/parmest/graphics.py +++ b/pyomo/contrib/parmest/graphics.py @@ -488,6 +488,99 @@ def pairwise_plot( plt.close() +def profile_likelihood_plot( + profile_results, alpha=None, filename=None, by="profiled_theta", y="lr_stat" +): + """ + Plot profile likelihood curves from Estimator.profile_likelihood results. + + Parameters + ---------- + profile_results: dict or DataFrame + Result dict returned by ``Estimator.profile_likelihood()`` or the + ``profiles`` DataFrame directly. + alpha: float, optional + If provided, draw the chi-square threshold line using dof=1. + filename: string, optional + Filename used to save the figure. + by: string, optional + Grouping column for separate panels. Default is ``profiled_theta``. + y: string, optional + Y-axis column to plot. Default is ``lr_stat``. + """ + assert isinstance(profile_results, (dict, pd.DataFrame)) + assert isinstance(alpha, (type(None), int, float)) + assert isinstance(filename, (type(None), str)) + assert isinstance(by, str) + assert isinstance(y, str) + + if isinstance(profile_results, dict): + if "profiles" not in profile_results: + raise KeyError("profile_results dict must contain 'profiles'.") + profiles = profile_results["profiles"] + else: + profiles = profile_results + + if not isinstance(profiles, pd.DataFrame): + raise TypeError("profiles must be a pandas DataFrame.") + if profiles.empty: + raise ValueError("profiles DataFrame is empty.") + required_cols = {"theta_value", y, by} + missing = required_cols.difference(profiles.columns) + if missing: + raise KeyError( + f"profiles DataFrame is missing required columns: {sorted(missing)}" + ) + + groups = list(profiles[by].dropna().unique()) + if len(groups) == 0: + raise ValueError(f"No non-null group values found in '{by}'.") + + fig, axes = plt.subplots( + nrows=len(groups), + ncols=1, + figsize=(6.5, max(3.0, 2.8 * len(groups))), + squeeze=False, + ) + + threshold = None + if alpha is not None: + threshold = stats.chi2.ppf(float(alpha), df=1) + + for i, grp in enumerate(groups): + ax = axes[i, 0] + subset = profiles[profiles[by] == grp].sort_values("theta_value") + ax.plot(subset["theta_value"], subset[y], marker="o", linewidth=1.5) + if "success" in subset.columns: + failed = subset[~subset["success"].astype(bool)] + if len(failed) > 0: + ax.scatter( + failed["theta_value"], + failed[y], + marker="x", + color="tab:red", + s=24, + label="failed", + ) + if threshold is not None: + ax.axhline(threshold, color="tab:orange", linestyle="--", linewidth=1.0) + ax.set_xlabel(str(grp)) + ax.set_ylabel(y) + ax.set_title(f"{by}: {grp}") + if ( + "success" in subset.columns + and len(subset[~subset["success"].astype(bool)]) > 0 + ): + ax.legend(loc="best", prop={"size": 8}) + + fig.tight_layout() + if filename is None: + plt.show() + else: + plt.savefig(filename) + plt.close() + + def fit_rect_dist(theta_values, alpha): """ Fit an alpha-level rectangular distribution to theta values diff --git a/pyomo/contrib/parmest/parmest.py b/pyomo/contrib/parmest/parmest.py index 21138d472da..ac539ebe9fa 100644 --- a/pyomo/contrib/parmest/parmest.py +++ b/pyomo/contrib/parmest/parmest.py @@ -11,6 +11,7 @@ #### Wrapping mpi-sppy functionality and local option Jan 2021, Feb 2021 #### Redesign with Experiment class Dec 2023 +# Options for using mpi-sppy or local EF only used in the deprecatedEstimator class # TODO: move use_mpisppy to a Pyomo configuration option # False implies always use the EF that is local to parmest use_mpisppy = True # Use it if we can but use local if not. @@ -41,6 +42,7 @@ import logging import types import json +import time from collections.abc import Callable from itertools import combinations from functools import singledispatchmethod @@ -80,6 +82,7 @@ logger = logging.getLogger(__name__) +# Only used in the deprecatedEstimator class def ef_nonants(ef): # Wrapper to call someone's ef_nonants # (the function being called is very short, but it might be changed) @@ -89,6 +92,7 @@ def ef_nonants(ef): return local_ef.ef_nonants(ef) +# Only used in the deprecatedEstimator class def _experiment_instance_creation_callback( scenario_name, node_names=None, cb_data=None ): @@ -297,6 +301,335 @@ def SSE_weighted(model): ) +def _validate_prior_FIM(prior_FIM, require_psd=True): + """ + Validate user-supplied prior Fisher Information Matrix. + + Parameters + ---------- + prior_FIM : pd.DataFrame + Prior Fisher Information Matrix with parameter names as both row and + column labels. + require_psd : bool, optional + If True, enforce positive semi-definiteness. Default is True. + """ + if not isinstance(prior_FIM, pd.DataFrame): + raise TypeError("prior_FIM must be a pandas DataFrame.") + + if prior_FIM.shape[0] != prior_FIM.shape[1]: + raise ValueError("prior_FIM must be square.") + + if set(prior_FIM.index) != set(prior_FIM.columns): + raise ValueError( + "prior_FIM row and column labels must match the same parameter names." + ) + + if not np.issubdtype(prior_FIM.values.dtype, np.number): + raise TypeError("prior_FIM entries must be numeric.") + + if not np.all(np.isfinite(prior_FIM.values)): + raise ValueError("prior_FIM entries must be finite.") + + if not np.allclose(prior_FIM.values, prior_FIM.values.T): + raise ValueError("prior_FIM must be symmetric.") + + if require_psd: + eigenvalues = np.linalg.eigvalsh(prior_FIM.values) + if np.any(eigenvalues < -1e-10): # tolerance for numerical noise + raise ValueError("prior_FIM must be positive semi-definite.") + + +def _calculate_L2_penalty(model, prior_FIM, theta_ref=None): + """ + Calculates (theta - theta_ref)^T * prior_FIM * (theta - theta_ref) + using label-based alignment for safety and subsets for efficiency. + + Parameters + ---------- + model : ConcreteModel + Annotated Pyomo model + prior_FIM : pd.DataFrame + Prior Fisher Information Matrix from previous experimental design + theta_ref: pd.Series, optional + Reference parameter values used in regularization. If None, defaults to the current parameter values in the model. + Returns + ------- + expr : Pyomo expression + Expression representing the L2-regularized objective addition + """ + _validate_prior_FIM(prior_FIM, require_psd=True) + + # Get current model parameters + # We assume model.unknown_parameters is a list of Pyomo Var objects + current_param_names = [p.name for p in model.unknown_parameters] + param_map = {p.name: p for p in model.unknown_parameters} + + # Confirm all matching parameters in both prior_FIM index and columns + common_params = [ + p + for p in current_param_names + if p in prior_FIM.columns and p in prior_FIM.index + ] + + if len(common_params) == 0: + + logger.warning( + "Warning: No matching parameters found between Model and Prior FIM. Returning standard objective." + ) + + return 0.0 # No regularization if no common parameters + + elif len(common_params) < len(set(prior_FIM.columns).union(set(prior_FIM.index))): + + logger.warning( + "Warning: Only a subset of parameters in the Prior FIM match the Model parameters. " + "Regularization will be applied only to the matching subset." + ) + else: + + logger.info( + "All parameters in the Prior FIM match the Model parameters. Regularization will be applied to all parameters." + ) + + # Slice the dataframes to ONLY the common subset + sub_FIM = prior_FIM.loc[common_params, common_params] + + # For the reference theta, we also subset to the common parameters. If theta_ref is None, we create a zero vector of the right size. + if theta_ref is not None: + if not isinstance(theta_ref, pd.Series): + theta_ref = pd.Series(theta_ref) + + missing_ref = [p for p in common_params if p not in theta_ref.index] + if missing_ref: + raise ValueError( + "theta_ref is missing values for parameter(s): " + + ", ".join(missing_ref) + ) + sub_theta = theta_ref.loc[common_params] + else: + sub_theta = pd.Series(0, index=common_params) + + # Fill the sub_theta with the initialized model parameter values (or zeros if not initialized) + for param in common_params: + sub_theta.loc[param] = pyo.value(param_map[param]) + logger.info( + "theta_ref is None. Using initialized parameter values as reference." + ) + + # Construct the Quadratic Form (The L2 term) + # @Reviewers: This can be calculated in a loop or in a vector form. + # Are there any issues with the vectorized form that we should be aware of for Pyomo expressions? + # The loop form is more transparent but the vectorized form is more efficient and concise. + + # l2_expr = 0.0 + # for i in common_params: + # di = name_to_var[i] - float(theta0.loc[i]) + # for j in common_params: + # qij = float(sub_FIM.loc[i, j]) + # if qij != 0.0: + # dj = name_to_var[j] - float(theta0.loc[j]) + # l2_expr += qij * di * dj + + # Create the (theta - theta_ref) vector, ensuring alignment by parameter name + delta_params = np.array( + [param_map[p] - sub_theta.loc[p] for p in common_params] + ) # (theta - theta_ref) vector + + # Compute the quadratic form: delta^T * FIM * delta + l2_term = delta_params.T @ sub_FIM.values @ delta_params + l2_term *= 0.5 # apply the 0.5 convention for quadratic regularization + + return l2_term + + +def L2_regularized_objective( + model, prior_FIM, theta_ref=None, regularization_weight=1.0, obj_function=SSE +): + """ + Calculates objective + regularization_weight*(theta - theta_ref)^T * prior_FIM * (theta - theta_ref) + using label-based alignment for safety and subsets for efficiency. + + Parameters + ---------- + model : ConcreteModel + Annotated Pyomo model + prior_FIM : pd.DataFrame + Prior Fisher Information Matrix from previous experimental design + theta_ref: pd.Series, optional + Reference parameter values used in regularization. If None, defaults to the current parameter values in the model. + regularization_weight: float, optional + Weighting factor for the regularization term. Default is 1.0. + obj_function: callable, optional + Built-in objective function selected by the user, e.g., `SSE`. Default is `SSE`. + + Returns + ------- + expr : Pyomo expression + Expression representing the L2-regularized objective + """ + + # Calculate the L2 penalty term + l2_term = _calculate_L2_penalty(model, prior_FIM, theta_ref) + + # Combine with objective + object_expr = obj_function(model) + + # Calculate the regularized objective + l2reg_objective = object_expr + (regularization_weight * l2_term) + + return l2reg_objective + + +def _calculate_L1_smooth_penalty( + model, prior_FIM, theta_ref=None, regularization_epsilon=1e-6 +): + """ + Calculates smooth L1 penalty: + sum_i w_i*sqrt((theta_i - theta_ref_i)^2 + regularization_epsilon) + where w_i is the corresponding diagonal entry in prior_FIM. + using label-based alignment for safety and subsets for efficiency. + + Parameters + ---------- + model : ConcreteModel + Annotated Pyomo model + prior_FIM : pd.DataFrame + Prior Fisher Information Matrix used for label-based variable selection + theta_ref: pd.Series, optional + Reference parameter values used in regularization. If None, defaults to + the current parameter values in the model. + regularization_epsilon: float, optional + Positive smoothing parameter in the smooth absolute value approximation. + + Returns + ------- + expr : Pyomo expression + Expression representing the smooth L1 objective addition + """ + _validate_prior_FIM(prior_FIM, require_psd=True) + + if regularization_epsilon <= 0: + raise ValueError("regularization_epsilon must be positive.") + + # Get current model parameters + current_param_names = [p.name for p in model.unknown_parameters] + param_map = {p.name: p for p in model.unknown_parameters} + + # Confirm all matching parameters in both prior_FIM index and columns + common_params = [ + p + for p in current_param_names + if p in prior_FIM.columns and p in prior_FIM.index + ] + + if len(common_params) == 0: + logger.warning( + "Warning: No matching parameters found between Model and Prior FIM. Returning standard objective." + ) + return 0.0 + elif len(common_params) < len(set(prior_FIM.columns).union(set(prior_FIM.index))): + logger.warning( + "Warning: Only a subset of parameters in the Prior FIM match the Model parameters. " + "Regularization will be applied only to the matching subset." + ) + else: + logger.info( + "All parameters in the Prior FIM match the Model parameters. Regularization will be applied to all parameters." + ) + + # Slice to matching subset and use diagonal entries as nonnegative weights. + sub_FIM = prior_FIM.loc[common_params, common_params] + + # Construct reference vector over the matching subset + if theta_ref is not None: + if not isinstance(theta_ref, pd.Series): + theta_ref = pd.Series(theta_ref) + missing_ref = [p for p in common_params if p not in theta_ref.index] + if missing_ref: + raise ValueError( + "theta_ref is missing values for parameter(s): " + + ", ".join(missing_ref) + ) + sub_theta = theta_ref.loc[common_params] + else: + sub_theta = pd.Series(0, index=common_params) + for param in common_params: + sub_theta.loc[param] = pyo.value(param_map[param]) + logger.info( + "theta_ref is None. Using initialized parameter values as reference." + ) + + l1_term = 0.0 + for p in common_params: + weight = float(sub_FIM.loc[p, p]) + var = param_map[p] + has_finite_lb = True + has_finite_ub = True + if hasattr(var, "lb") and hasattr(var, "ub"): + lb = pyo.value(var.lb, exception=False) if var.lb is not None else None + ub = pyo.value(var.ub, exception=False) if var.ub is not None else None + has_finite_lb = lb is not None and np.isfinite(lb) + has_finite_ub = ub is not None and np.isfinite(ub) + if weight == 0.0: + if not has_finite_lb and not has_finite_ub: + logger.warning( + "L1 regularization weight is zero for parameter '%s' and the " + "parameter has no finite bounds. This can create a weakly " + "identified flat direction.", + p, + ) + continue + delta = var - sub_theta.loc[p] + l1_term += weight * pyo.sqrt(delta * delta + regularization_epsilon) + + return l1_term + + +def L1_regularized_objective( + model, + prior_FIM, + theta_ref=None, + regularization_weight=1.0, + regularization_epsilon=1e-6, + obj_function=SSE, +): + """ + Calculates objective + regularization_weight*sum_i w_i*sqrt((theta_i-theta_ref_i)^2 + regularization_epsilon) + where w_i is the corresponding diagonal entry in prior_FIM. + using label-based alignment for safety and subsets for efficiency. + + Parameters + ---------- + model : ConcreteModel + Annotated Pyomo model + prior_FIM : pd.DataFrame + Prior Fisher Information Matrix used for label-based variable selection + theta_ref: pd.Series, optional + Reference parameter values used in regularization. If None, defaults to + the current parameter values in the model. + regularization_weight: float, optional + Weighting factor for the regularization term. Default is 1.0. + regularization_epsilon: float, optional + Positive smoothing parameter in the smooth absolute value approximation. + obj_function: callable, optional + Built-in objective function selected by the user, e.g., `SSE`. Default is `SSE`. + + Returns + ------- + expr : Pyomo expression + Expression representing the smooth L1-regularized objective + """ + l1_term = _calculate_L1_smooth_penalty( + model, + prior_FIM=prior_FIM, + theta_ref=theta_ref, + regularization_epsilon=regularization_epsilon, + ) + object_expr = obj_function(model) + l1reg_objective = object_expr + (regularization_weight * l1_term) + return l1reg_objective + + def _check_model_labels(model): """ Checks if the annotated Pyomo model contains the necessary suffixes @@ -354,14 +687,24 @@ def _count_total_experiments(experiment_list): Returns ------- - total_number_data : int + total_data_points : int The total number of data points in the list of experiments """ - total_number_data = 0 + total_data_points = 0 for experiment in experiment_list: - total_number_data += len(experiment.get_labeled_model().experiment_outputs) + # 1. Identify the first parent component of the experiment outputs + output_vars = experiment.get_labeled_model().experiment_outputs + + # 1. Identify the first parent component + first_var_key = list(output_vars.keys())[0] + first_parent = first_var_key.parent_component() + # 2. Count only the keys that belong to this specific parent + first_parent_indices = [ + v for v in output_vars.keys() if v.parent_component() is first_parent + ] + total_data_points += len(first_parent_indices) - return total_number_data + return total_data_points class CovarianceMethod(Enum): @@ -375,6 +718,11 @@ class ObjectiveType(Enum): SSE_weighted = "SSE_weighted" +class RegularizationType(Enum): + L2 = "L2" + L1 = "L1" + + # Compute the Jacobian matrix of measured variables with respect to the parameters def _compute_jacobian(experiment, theta_vals, step, solver, tee): """ @@ -474,6 +822,8 @@ def compute_covariance_matrix( solver, tee, estimated_var=None, + prior_FIM=None, + regularization_weight=1.0, ): """ Computes the covariance matrix of the estimated parameters using @@ -502,7 +852,13 @@ def compute_covariance_matrix( Value of the estimated variance of the measurement error in cases where the user does not supply the measurement error standard deviation - + prior_FIM: pd.DataFrame, optional + Prior Fisher Information Matrix from previous experimental design + to be added to the FIM of the current experiments for covariance estimation. + The prior_FIM should be a square matrix with parameter names as both + row and column labels. + regularization_weight: float, optional + Weighting factor for the regularization term. Default is 1.0. Returns ------- cov : pd.DataFrame @@ -540,6 +896,24 @@ def compute_covariance_matrix( FIM = np.sum(FIM_all_exp, axis=0) + # Add prior_FIM if including regularization. We expand the prior FIM to match the size of the + # FIM and align it based on parameter names to ensure correct addition. + # Add the prior FIM to the FIM of the current experiments, weighted by the regularization weight + if prior_FIM is not None: + if regularization_weight < 0: + raise ValueError("regularization_weight must be nonnegative.") + + expanded_prior_FIM = _expand_prior_FIM( + experiment_list[0], prior_FIM # theta_vals + ) # sanity check and alignment + + # Check that the prior FIM shape is the same as the FIM shape + if expanded_prior_FIM.shape != FIM.shape: + raise ValueError( + "The shape of the prior FIM must be the same as the shape of the FIM." + ) + FIM += expanded_prior_FIM * regularization_weight + # calculate the covariance matrix try: cov = np.linalg.inv(FIM) @@ -745,6 +1119,40 @@ def _kaug_FIM(experiment, obj_function, theta_vals, solver, tee, estimated_var=N return FIM +def _expand_prior_FIM(experiment, prior_FIM): + """ + Expands the prior FIM to match the size of the FIM of the current experiment + + Parameters + ---------- + experiment : Experiment class + Experiment class object that contains the Pyomo model + for a particular experimental condition + prior_FIM : pd.DataFrame + Prior Fisher Information Matrix from previous experimental design + + Returns + ------- + expanded_prior_FIM : pd.DataFrame + Expanded prior FIM with the same size as the FIM of the current experiment + """ + _validate_prior_FIM(prior_FIM, require_psd=True) + + model = _get_labeled_model(experiment) + + # Extract parameter names from the Pyomo model + param_names = [param.name for param in model.unknown_parameters] + + # 1. Expand Prior FIM + # We reindex to match param_names. Parameters not in prior_FIM + # will be filled with 0, which maintains the PSD property. + expanded_prior_FIM = prior_FIM.reindex( + index=param_names, columns=param_names, fill_value=0.0 + ) + + return expanded_prior_FIM + + class Estimator: """ Parameter estimation class @@ -768,6 +1176,28 @@ class Estimator: solver_options: dict, optional Provides options to the solver (also the name of an attribute). Default is None. + + Added keyword arguments for objective regularization: + regularization: string, optional + Built-in regularization type ("L2" or "L1"). If no regularization is + specified, no regularization term is added to the objective. + Default is None. + prior_FIM: pd.DataFrame, optional + Prior Fisher Information Matrix from previous experimental design + to be added to the FIM of the current experiments for regularization. + The prior_FIM should be a square matrix with parameter names as both + row and column labels. + theta_ref: pd.Series, optional + Reference parameter values used in regularization. + If None, defaults to the current parameter values in the model. + regularization_weight: float, optional + Weighting factor for the regularization term. Used with + ``regularization="L2"`` or ``regularization="L1"`` + and defaults to 1.0 when omitted. + regularization_epsilon: float, optional + Positive smoothing parameter used with ``regularization="L1"`` + in ``sqrt((theta-theta_ref)^2 + regularization_epsilon)``. + Defaults to ``1e-6`` when omitted. """ # The singledispatchmethod decorator is used here as a deprecation @@ -780,6 +1210,11 @@ def __init__( self, experiment_list, obj_function=None, + regularization=None, + prior_FIM=None, + theta_ref=None, + regularization_weight=None, + regularization_epsilon=None, tee=False, diagnostic_mode=False, solver_options=None, @@ -810,10 +1245,91 @@ def __init__( else: self.obj_function = obj_function + if isinstance(regularization, str): + try: + self.regularization = RegularizationType(regularization) + except ValueError: + raise ValueError( + f"Invalid regularization type: '{regularization}'. " + f"Choose from: {[e.value for e in RegularizationType]}." + ) + elif isinstance(regularization, RegularizationType): + self.regularization = regularization + elif regularization is None: + self.regularization = None + else: + raise TypeError( + "regularization must be None or one of " + f"{[e.value for e in RegularizationType]}." + ) + self.tee = tee self.diagnostic_mode = diagnostic_mode self.solver_options = solver_options + # Validate regularization options and defaults + if self.regularization is None: + if ( + prior_FIM is not None + or theta_ref is not None + or regularization_weight is not None + or regularization_epsilon is not None + ): + raise ValueError( + "regularization must be set when supplying prior_FIM, theta_ref, " + "regularization_weight, or regularization_epsilon." + ) + self.prior_FIM = None + self.theta_ref = None + self.regularization_weight = None + self.regularization_epsilon = None + elif self.regularization == RegularizationType.L2: + if prior_FIM is None: + raise ValueError("prior_FIM must be provided when regularization='L2'.") + _validate_prior_FIM(prior_FIM, require_psd=True) + if theta_ref is not None and not isinstance(theta_ref, pd.Series): + theta_ref = pd.Series(theta_ref) + if regularization_epsilon is not None: + raise ValueError( + "regularization_epsilon is only supported when regularization='L1'." + ) + + if regularization_weight is None: + regularization_weight = 1.0 + if regularization_weight < 0: + raise ValueError("regularization_weight must be nonnegative.") + + self.prior_FIM = prior_FIM + self.theta_ref = theta_ref + self.regularization_weight = regularization_weight + self.regularization_epsilon = None + elif self.regularization == RegularizationType.L1: + if prior_FIM is None: + raise ValueError("prior_FIM must be provided when regularization='L1'.") + _validate_prior_FIM(prior_FIM, require_psd=True) + if theta_ref is not None and not isinstance(theta_ref, pd.Series): + theta_ref = pd.Series(theta_ref) + + if regularization_weight is None: + regularization_weight = 1.0 + if regularization_weight < 0: + raise ValueError("regularization_weight must be nonnegative.") + + if regularization_epsilon is None: + regularization_epsilon = 1e-6 + if regularization_epsilon <= 0: + raise ValueError("regularization_epsilon must be positive.") + + self.prior_FIM = prior_FIM + self.theta_ref = theta_ref + self.regularization_weight = regularization_weight + self.regularization_epsilon = regularization_epsilon + else: + raise ValueError( + f"Unsupported regularization option: {self.regularization}. " + f"Choose from {[e.value for e in RegularizationType]}." + ) + # TODO: delete this when the deprecated interface is removed self.pest_deprecated = None @@ -909,9 +1425,91 @@ def _expand_indexed_unknowns(self, model_temp): return model_theta_list + # Reviewers: Put in architecture to calculate a regularization weight based on the current parameter values and the prior FIM. + # However, if the prior_FIM is properly defined, this should not be necessary. Are there any use cases for this where we should give + # the user a scaling option, or remove and trust the prior_FIM to be properly scaled? + + # def _calc_regularization_weight(self, solver='ipopt'): + # """ + # Calculate regularization weight as the ratio of the objective value to the L2 term value at the current parameter values to balance their magnitudes. + # """ + # # Solve the model at the current parameter values to get the objective function value + # sse_vals = [] + # for experiment in self.exp_list: + # model = _get_labeled_model(experiment) + + # # fix the value of the unknown parameters to the estimated values + # for param in model.unknown_parameters: + # param.fix(pyo.value(param)) + + # # re-solve the model with the estimated parameters + # results = pyo.SolverFactory(solver).solve(model, tee=self.tee) + # assert_optimal_termination(results) + + # # choose and evaluate the sum of squared errors expression + # if self.obj_function == ObjectiveType.SSE: + # sse_expr = SSE(model) + # elif self.obj_function == ObjectiveType.SSE_weighted: + # sse_expr = SSE_weighted(model) + # else: + # raise ValueError( + # f"Invalid objective function for covariance calculation. " + # f"The covariance matrix can only be calculated using the built-in " + # f"objective functions: {[e.value for e in ObjectiveType]}. Supply " + # f"the Estimator object one of these built-in objectives and " + # f"re-run the code." + # ) + # l2_expr = _calculate_L2_penalty(model, self.prior_FIM, self.theta_ref) + + # # evaluate the numerical SSE and store it + # sse_val = pyo.value(sse_expr) + # sse_vals.append(sse_val) + + # sse = sum(sse_vals) + + # if l2_expr is None: + # l2_value = 0 + # else: + # l2_value = pyo.value(l2_expr) + + # if l2_value == 0: + # logger.warning( + # "L2 penalty is zero at the current parameter values. Regularization weight set to 1.0 by default." + # ) + # return 1.0 + + # reg_weight = float(pyo.value(sse) / (pyo.value(l2_value))) + # logger.info(f"Calculated regularization weight: {reg_weight}") + # return reg_weight + def _create_parmest_model(self, experiment_number): """ - Modify the Pyomo model for parameter estimation + Build a parmest-ready model for a single experiment. + + This helper retrieves the labeled experiment model, prepares objective + components needed by parmest, and converts unknown parameters to + decision variables. The returned model is the one used to populate EF + scenario blocks. + + Parameters + ---------- + experiment_number : int + Index into ``self.exp_list`` selecting which experiment model to + load. + + Returns + ------- + ConcreteModel + A model configured for parmest optimization, including: + 1. a ``Total_Cost_Objective`` (if ``self.obj_function`` is set) + 2. converted unknown-parameter variables (unfixed) + + Notes + ----- + - Existing user objectives are deactivated before parmest objective + components are attached. + - Reserved component names are checked to avoid overriding user model + components. """ model = _get_labeled_model(self.exp_list[experiment_number]) @@ -939,15 +1537,45 @@ def _create_parmest_model(self, experiment_number): # TODO, this needs to be turned into an enum class of options that still support # custom functions - if self.obj_function is ObjectiveType.SSE: - second_stage_rule = SSE - self.covariance_objective = second_stage_rule - elif self.obj_function is ObjectiveType.SSE_weighted: - second_stage_rule = SSE_weighted - self.covariance_objective = second_stage_rule + + if isinstance(self.obj_function, ObjectiveType): + + if self.obj_function == ObjectiveType.SSE: + self.covariance_objective = SSE + + elif self.obj_function == ObjectiveType.SSE_weighted: + self.covariance_objective = SSE_weighted + + base_objective = self.covariance_objective else: # A custom function uses model.experiment_outputs as data - second_stage_rule = self.obj_function + self.covariance_objective = None + base_objective = self.obj_function + + if self.regularization is None: + second_stage_rule = base_objective + elif self.regularization == RegularizationType.L2: + second_stage_rule = lambda m: L2_regularized_objective( + m, + prior_FIM=self.prior_FIM, + theta_ref=self.theta_ref, + regularization_weight=self.regularization_weight, + obj_function=base_objective, + ) + elif self.regularization == RegularizationType.L1: + second_stage_rule = lambda m: L1_regularized_objective( + m, + prior_FIM=self.prior_FIM, + theta_ref=self.theta_ref, + regularization_weight=self.regularization_weight, + regularization_epsilon=self.regularization_epsilon, + obj_function=base_objective, + ) + else: + raise ValueError( + f"Unsupported regularization option: {self.regularization}. " + f"Choose from {[e.value for e in RegularizationType]}." + ) model.FirstStageCost = pyo.Expression(expr=0) model.SecondStageCost = pyo.Expression(rule=second_stage_rule) @@ -969,197 +1597,500 @@ def _instance_creation_callback(self, experiment_number=None, cb_data=None): model = self._create_parmest_model(experiment_number) return model - def _Q_opt( + def _create_scenario_blocks( self, - ThetaVals=None, - solver="ef_ipopt", - return_values=[], bootlist=None, - calc_cov=NOTSET, - cov_n=NOTSET, + theta_vals=None, + fix_theta=False, + multistart=False, + fixed_theta_values=None, ): """ - Set up all thetas as first stage Vars, return resulting theta - values as well as the objective function value. + Build the block-based extensive form (EF) model for estimation. - """ - if solver == "k_aug": - raise RuntimeError("k_aug no longer supported.") + The EF includes: + 1. a master theta variable container (``model.parmest_theta``), + 2. one child block per selected experiment (``model.exp_scenarios``), + 3. optional theta-linking constraints between master and child blocks, + 4. a single aggregate objective over all child blocks. - # (Bootstrap scenarios will use indirection through the bootlist) - if bootlist is None: - scenario_numbers = list(range(len(self.exp_list))) - scen_names = ["Scenario{}".format(i) for i in scenario_numbers] - else: - scen_names = ["Scenario{}".format(i) for i in range(len(bootlist))] + In multistart mode, this method also refreshes experiment-level cached + model state before rebuilding each scenario so per-start initializations + are applied to the model that is actually solved. - # get the probability constant that is applied to the objective function - # parmest solves the estimation problem by applying equal probabilities to - # the objective function of all the scenarios from the experiment list - self.obj_probability_constant = len(scen_names) + Parameters + ---------- + bootlist : list, optional + Experiment indices to include. If ``None``, all experiments in + ``self.exp_list`` are used. + theta_vals : dict, optional + Theta values to apply as initial values to parent and child theta + variables. When ``multistart=True``, these values are also pushed to + experiment ``theta_initial`` (if present) before rebuilding. + fix_theta : bool, optional + If ``True``, theta variables are fixed in each scenario and no + linking constraints are created. + multistart : bool, optional + If ``True``, force experiment model refresh between starts to avoid + stale cached model reuse. - # tree_model.CallbackModule = None - outer_cb_data = dict() - outer_cb_data["callback"] = self._instance_creation_callback - if ThetaVals is not None: - outer_cb_data["ThetaVals"] = ThetaVals - if bootlist is not None: - outer_cb_data["BootList"] = bootlist - outer_cb_data["cb_data"] = None # None is OK - outer_cb_data["theta_names"] = self.estimator_theta_names + Returns + ------- + ConcreteModel + EF model used by parmest solve routines. - options = {"solver": "ipopt"} - scenario_creator_options = {"cb_data": outer_cb_data} - if use_mpisppy: - ef = sputils.create_EF( - scen_names, - _experiment_instance_creation_callback, - EF_name="_Q_opt", - suppress_warnings=True, - scenario_creator_kwargs=scenario_creator_options, - ) - else: - ef = local_ef.create_EF( - scen_names, - _experiment_instance_creation_callback, - EF_name="_Q_opt", - suppress_warnings=True, - scenario_creator_kwargs=scenario_creator_options, + Raises + ------ + ValueError + If the selected scenario set is empty. + """ + # Build a clean parent EF container and attach one scenario model per block. + model = pyo.ConcreteModel() + if multistart: + template_experiment = self.exp_list[0] + if theta_vals is not None and hasattr(template_experiment, "theta_initial"): + template_experiment.theta_initial = dict(theta_vals) + if hasattr(template_experiment, "model"): + template_experiment.model = None + template_model = self._create_parmest_model(0) + expanded_theta_names = self._expand_indexed_unknowns(template_model) + model._parmest_theta_names = tuple(expanded_theta_names) + model.parmest_theta = pyo.Var(model._parmest_theta_names) + + fixed_theta_values = fixed_theta_values or {} + invalid_fixed_theta = set(fixed_theta_values).difference(expanded_theta_names) + if invalid_fixed_theta: + raise ValueError( + f"Unknown theta name(s) in fixed_theta_values: {sorted(invalid_fixed_theta)}" ) - self.ef_instance = ef + fixed_theta_names = set(fixed_theta_values.keys()) + + for name in expanded_theta_names: + template_theta_var = template_model.find_component(name) + parent_theta_var = model.parmest_theta[name] + parent_theta_var.set_value(pyo.value(template_theta_var)) + if theta_vals is not None and name in theta_vals: + parent_theta_var.set_value(theta_vals[name]) + if name in fixed_theta_values: + parent_theta_var.set_value(fixed_theta_values[name]) + if fix_theta: + parent_theta_var.fix() + elif name in fixed_theta_names: + parent_theta_var.fix() + else: + parent_theta_var.unfix() - # Solve the extensive form with ipopt - if solver == "ef_ipopt": - if calc_cov is NOTSET or not calc_cov: - # Do not calculate the reduced hessian + # Set the number of experiments to use, either from bootlist or all experiments + scenario_numbers = ( + list(bootlist) if bootlist is not None else list(range(len(self.exp_list))) + ) + self.obj_probability_constant = len(scenario_numbers) + if self.obj_probability_constant <= 0: + raise ValueError("At least one scenario is required to build the EF model.") + + # Create indexed block for holding scenario models + model.exp_scenarios = pyo.Block(range(self.obj_probability_constant)) + for i, experiment_number in enumerate(scenario_numbers): + if multistart: + experiment = self.exp_list[experiment_number] + if theta_vals is not None and hasattr(experiment, "theta_initial"): + experiment.theta_initial = dict(theta_vals) + if hasattr(experiment, "model"): + experiment.model = None + parmest_model = self._create_parmest_model(experiment_number) + for name in expanded_theta_names: + child_theta_var = parmest_model.find_component(name) + parent_theta_var = model.parmest_theta[name] + if theta_vals is not None and name in theta_vals: + child_theta_var.set_value(theta_vals[name]) + else: + child_theta_var.set_value(pyo.value(parent_theta_var)) + if name in fixed_theta_values: + child_theta_var.set_value(fixed_theta_values[name]) + if fix_theta: + child_theta_var.fix() + elif name in fixed_theta_names: + child_theta_var.fix() + else: + child_theta_var.unfix() + model.exp_scenarios[i].transfer_attributes_from(parmest_model) + + model.theta_link_constraints = pyo.ConstraintList() + if not fix_theta: + for name in expanded_theta_names: + if name in fixed_theta_names: + continue + parent_theta_var = model.parmest_theta[name] + for i in range(self.obj_probability_constant): + child_theta_var = model.exp_scenarios[i].find_component(name) + model.theta_link_constraints.add( + child_theta_var == parent_theta_var + ) - solver = SolverFactory('ipopt') - if self.solver_options is not None: - for key in self.solver_options: - solver.options[key] = self.solver_options[key] + for block in model.exp_scenarios.values(): + for obj in block.component_objects(pyo.Objective): + obj.deactivate() - solve_result = solver.solve(self.ef_instance, tee=self.tee) - assert_optimal_termination(solve_result) - elif calc_cov is not NOTSET and calc_cov: - # parmest makes the fitted parameters stage 1 variables - ind_vars = [] - for nd_name, Var, sol_val in ef_nonants(ef): - ind_vars.append(Var) - # calculate the reduced hessian - solve_result, inv_red_hes = ( - inverse_reduced_hessian.inv_reduced_hessian_barrier( - self.ef_instance, - independent_variables=ind_vars, - solver_options=self.solver_options, - tee=self.tee, - ) + # Make an objective that sums over all scenario blocks and divides by number of experiments + def total_obj(m): + return ( + sum( + block.Total_Cost_Objective.expr + for block in m.exp_scenarios.values() ) + / self.obj_probability_constant + ) - if self.diagnostic_mode: - print( - ' Solver termination condition = ', - str(solve_result.solver.termination_condition), - ) + model.Obj = pyo.Objective(rule=total_obj, sense=pyo.minimize) + self.ef_instance = model + return model - # assume all first stage are thetas... - theta_vals = {} - for nd_name, Var, sol_val in ef_nonants(ef): - # process the name - # the scenarios are blocks, so strip the scenario name - var_name = Var.name[Var.name.find(".") + 1 :] - theta_vals[var_name] = sol_val + def _generate_initial_theta( + self, + seed=None, + n_restarts=None, + multistart_sampling_method=None, + user_provided_df=None, + experiment_number=0, + ): + """ + Create the canonical multistart initialization/results DataFrame. - obj_val = pyo.value(ef.EF_Obj) - self.obj_value = obj_val - self.estimated_theta = theta_vals + Output schema is: + 1. theta columns (canonical order, quote-normalized names), + 2. ``converged_`` columns, + 3. ``final objective``, ``solver termination``, ``solve_time``. - if calc_cov is not NOTSET and calc_cov: - # Calculate the covariance matrix + Initial theta rows are either sampled from bounds or taken from a + user-provided DataFrame. - if not isinstance(cov_n, int): - raise TypeError( - f"Expected an integer for the 'cov_n' argument. " - f"Got {type(cov_n)}." - ) - num_unknowns = max( - [ - len(experiment.get_labeled_model().unknown_parameters) - for experiment in self.exp_list - ] - ) - assert cov_n > num_unknowns, ( - "The number of datapoints must be greater than the " - "number of parameters to estimate." - ) + Parameters + ---------- + seed : int, optional + Random seed used by stochastic samplers. + n_restarts : int, optional + Number of starts to generate for sampled methods. Ignored when + ``user_provided_df`` is provided. + multistart_sampling_method : str, optional + Sampling method. Supported values: + ``uniform_random``, ``latin_hypercube``, ``sobol_sampling``. + user_provided_df : DataFrame, optional + Explicit initialization table. Must contain exactly the theta + columns (order may vary). Values must be finite and within bounds. + experiment_number : int, optional + Experiment index used to discover canonical theta names and bounds. - # Number of data points considered - n = cov_n + Returns + ------- + DataFrame + Canonical initialization/results table ready for multistart solve + bookkeeping. + + Raises + ------ + ValueError + For missing/invalid bounds, invalid sampling method, malformed + user-provided starts, non-finite values, or out-of-bound starts. + TypeError + For invalid input types (for example, non-DataFrame + ``user_provided_df`` or non-integer ``n_restarts`` when required). + RuntimeError + If expected theta components cannot be located on the model. + """ + parmest_model = self._create_parmest_model(experiment_number) - # Extract number of fitted parameters - l = len(theta_vals) + raw_theta_names = self._expand_indexed_unknowns(parmest_model) + theta_names = [n.replace("'", "") for n in raw_theta_names] + if len(theta_names) != len(set(theta_names)): + raise ValueError(f"Duplicate theta names are not allowed: {theta_names}") - # Assumption: Objective value is sum of squared errors - sse = obj_val + theta_vars = [parmest_model.find_component(name) for name in raw_theta_names] + if any(v is None for v in theta_vars): + raise RuntimeError( + "Failed to locate one or more theta components on model." + ) - '''Calculate covariance assuming experimental observation errors - are independent and follow a Gaussian distribution - with constant variance. + lower_bound = np.array([v.lb for v in theta_vars], dtype=float) + upper_bound = np.array([v.ub for v in theta_vars], dtype=float) - The formula used in parmest was verified against equations - (7-5-15) and (7-5-16) in "Nonlinear Parameter Estimation", - Y. Bard, 1974. + if np.any(np.isnan(lower_bound)) or np.any(np.isnan(upper_bound)): + raise ValueError( + "The lower and upper bounds for the theta values must be defined." + ) + if np.any(lower_bound > upper_bound): + raise ValueError( + "Each lower bound must be less than or equal to the corresponding upper bound." + ) - This formula is also applicable if the objective is scaled by a - constant; the constant cancels out. - (was scaled by 1/n because it computes an expected value.) - ''' - cov = 2 * sse / (n - l) * inv_red_hes - cov = pd.DataFrame( - cov, index=theta_vals.keys(), columns=theta_vals.keys() + if user_provided_df is not None: + if not isinstance(user_provided_df, pd.DataFrame): + raise TypeError("user_provided_df must be a pandas DataFrame.") + if user_provided_df.shape[1] != len(theta_names): + raise ValueError( + "user_provided_df must have exactly one column per theta name." + ) + clean_cols = [str(c).replace("'", "") for c in user_provided_df.columns] + if len(clean_cols) != len(set(clean_cols)): + raise ValueError("Duplicate theta columns are not allowed.") + if set(clean_cols) != set(theta_names): + raise ValueError( + f"Provided columns {clean_cols} do not match expected theta names {theta_names}." + ) + df_multistart = user_provided_df.copy() + df_multistart.columns = clean_cols + df_multistart = df_multistart.reindex(columns=theta_names) + if df_multistart.shape[0] == 0: + raise ValueError("user_provided_df must contain at least one row.") + if n_restarts is not None and n_restarts != df_multistart.shape[0]: + raise ValueError( + "n_restarts must match the number of rows in user_provided_df." ) + theta_vals_multistart = df_multistart.to_numpy(dtype=float) + n_restarts = df_multistart.shape[0] + elif multistart_sampling_method == "uniform_random": + if not isinstance(n_restarts, int): + raise TypeError("n_restarts must be an integer.") + if n_restarts <= 0: + raise ValueError("n_restarts must be greater than zero.") + # Use a local RNG to avoid mutating global random state. + rng = np.random.default_rng(seed) + theta_vals_multistart = rng.uniform( + low=lower_bound, high=upper_bound, size=(n_restarts, len(theta_names)) + ) - theta_vals = pd.Series(theta_vals) + elif multistart_sampling_method == "latin_hypercube": + if not isinstance(n_restarts, int): + raise TypeError("n_restarts must be an integer.") + if n_restarts <= 0: + raise ValueError("n_restarts must be greater than zero.") + # Generate theta values using Latin hypercube sampling or Sobol sampling + # Generate theta values using Latin hypercube sampling + # Create a Latin Hypercube sampler that uses the dimensions of the theta names + sampler = scipy.stats.qmc.LatinHypercube(d=len(theta_names), seed=seed) + # Generate random samples in the range of [0, 1] for number of restarts + samples = sampler.random(n=n_restarts) + # Resulting samples should be size (n_restarts, len(theta_names)) + + elif multistart_sampling_method == "sobol_sampling": + if not isinstance(n_restarts, int): + raise TypeError("n_restarts must be an integer.") + if n_restarts <= 0: + raise ValueError("n_restarts must be greater than zero.") + sampler = scipy.stats.qmc.Sobol(d=len(theta_names), seed=seed) + # Generate theta values using Sobol sampling + # The first value of the Sobol sequence is 0, so we skip it + samples = sampler.random(n=n_restarts + 1)[1:] + + elif multistart_sampling_method == "user_provided_values": + raise ValueError( + "multistart_sampling_method='user_provided_values' requires user_provided_df." + ) - if len(return_values) > 0: - var_values = [] - if len(scen_names) > 1: # multiple scenarios - block_objects = self.ef_instance.component_objects( - Block, descend_into=False - ) - else: # single scenario - block_objects = [self.ef_instance] - for exp_i in block_objects: - vals = {} - for var in return_values: - exp_i_var = exp_i.find_component(str(var)) - if ( - exp_i_var is None - ): # we might have a block such as _mpisppy_data - continue - # if value to return is ContinuousSet - if type(exp_i_var) == ContinuousSet: - temp = list(exp_i_var) - else: - temp = [pyo.value(_) for _ in exp_i_var.values()] - if len(temp) == 1: - vals[var] = temp[0] - else: - vals[var] = temp - if len(vals) > 0: - var_values.append(vals) - var_values = pd.DataFrame(var_values) - if calc_cov is not NOTSET and calc_cov: - return obj_val, theta_vals, var_values, cov - elif calc_cov is NOTSET or not calc_cov: - return obj_val, theta_vals, var_values + else: + raise ValueError( + "Invalid sampling method. Choose 'uniform_random', 'latin_hypercube', 'sobol_sampling' or 'user_provided_values'." + ) + + if ( + multistart_sampling_method == "sobol_sampling" + or multistart_sampling_method == "latin_hypercube" + ): + # Scale the samples to the range of the lower and upper bounds for each theta in theta_names + # The samples are in the range [0, 1], so we scale them to the range of the lower and upper bounds + theta_vals_multistart = np.array( + [lower_bound + (upper_bound - lower_bound) * theta for theta in samples] + ) + + # Create a DataFrame where each row is an initial theta vector for a restart + if user_provided_df is None: + # Ensure theta_vals_multistart is 2D (n_restarts, len(theta_names)) + arr = np.atleast_2d(theta_vals_multistart) + if arr.shape[0] == 1 and n_restarts > 1: + arr = np.tile(arr, (n_restarts, 1)) + df_multistart = pd.DataFrame(arr, columns=theta_names) + + theta_arr = df_multistart[theta_names].to_numpy(dtype=float) + if not np.isfinite(theta_arr).all(): + raise ValueError("Initial theta values must be finite.") + if np.any(theta_arr < lower_bound) or np.any(theta_arr > upper_bound): + raise ValueError("Initial theta values must be within model bounds.") + + # Add columns for output info, initialized as nan + for name in theta_names: + df_multistart[f'converged_{name}'] = np.nan + df_multistart["final objective"] = np.nan + df_multistart["solver termination"] = "" + df_multistart["solve_time"] = np.nan + + # Debugging output + # print(df_multistart) + + return df_multistart + + # Redesigned _Q_opt method using scenario blocks, and combined with + # _Q_at_theta structure. + def _Q_opt( + self, + return_values=None, + bootlist=None, + solver="ef_ipopt", + theta_vals=None, + fix_theta=False, + multistart=False, + fixed_theta_values=None, + ): + """ + Solve the EF parameter-estimation problem and return objective/theta data. - if calc_cov is not NOTSET and calc_cov: - return obj_val, theta_vals, cov - elif calc_cov is NOTSET or not calc_cov: - return obj_val, theta_vals + This routine creates the EF model via ``_create_scenario_blocks``, + solves it with the requested solver, and returns objective value plus + theta estimates. Depending on mode, it can also return variable values + and covariance estimates. + Parameters + ---------- + return_values : list, optional + List of variable names to return values for. Default is None. + bootlist : list, optional + List of bootstrap experiment numbers to use. If None, use all experiments in exp_list. + Default is None. + theta_vals : dict, optional + Dictionary of theta values to set in the model. If None, use default values from experiment class. + Default is None. + solver : str, optional + Solver to use for optimization. Default is "ef_ipopt". + calc_cov : bool, optional + If True, calculate covariance matrix of estimated parameters. Default is NOTSET. + cov_n : int, optional + Number of data points to use for covariance calculation. Required if calc_cov is True. Default is NOTSET. + fix_theta : bool, optional + If True, fix the theta values in the model. If False, leave them free. + Default is False. + multistart : bool, optional + If True, run in multistart mode. Non-optimal termination is + returned instead of raising assertion failure. + + Returns + ------- + tuple + Return shape depends on mode: + 1. Standard solve: ``(obj_value, theta_series)`` + 2. Standard + return values: ``(obj_value, theta_series, var_values)`` + 3. Standard + covariance: ``(obj_value, theta_series, cov)`` or + ``(obj_value, theta_series, var_values, cov)`` + 4. Fixed-theta or multistart: ``(obj_value, theta_dict, worst_status)`` + """ + # Create extended form model with scenario blocks + model = self._create_scenario_blocks( + bootlist=bootlist, + theta_vals=theta_vals, + fix_theta=fix_theta, + fixed_theta_values=fixed_theta_values, + multistart=multistart, + ) + expanded_theta_names = list(model._parmest_theta_names) + + # Print model if in diagnostic mode + if self.diagnostic_mode: + print("Parmest _Q_opt model with scenario blocks:") + model.pprint() + + # Check solver and set options + if solver == "k_aug": + raise RuntimeError("k_aug no longer supported.") + if solver == "ef_ipopt": + sol = SolverFactory('ipopt') else: raise RuntimeError("Unknown solver in Q_Opt=" + solver) + # Currently, parmest is only tested with ipopt via ef_ipopt + # No other pyomo solvers have been verified to work with parmest from current release + # to my knowledge. + + if self.solver_options is not None: + for key in self.solver_options: + sol.options[key] = self.solver_options[key] + + # Solve model + solve_result = sol.solve(model, tee=self.tee) + partial_fix_mode = bool(fixed_theta_values) + + # Separate handling of termination conditions for _Q_at_theta vs _Q_opt + # If not fixing theta, ensure optimal termination of the solve to return result + if not fix_theta and not multistart and not partial_fix_mode: + # Ensure optimal termination + assert_optimal_termination(solve_result) + # If fixing theta, capture termination condition if not optimal unless infeasible + else: + # Initialize worst_status to optimal, update if not optimal + worst_status = pyo.TerminationCondition.optimal + # Get termination condition from solve result + status = solve_result.solver.termination_condition + + # In case of fixing theta, just log a warning if not optimal + if status != pyo.TerminationCondition.optimal: + # logger.warning( + # "Solver did not terminate optimally when thetas were fixed. " + # "Termination condition: %s", + # str(status), + # ) + # Unless infeasible, update worst_status + if worst_status != pyo.TerminationCondition.infeasible: + worst_status = status + + # Extract objective value + obj_value = pyo.value(model.Obj) + theta_estimates = {} + # Extract theta estimates from parent model + for name in expanded_theta_names: + theta_estimates[name] = pyo.value(model.parmest_theta[name]) + + self.obj_value = obj_value + self.estimated_theta = theta_estimates + + # If fixing theta, return objective value, theta estimates, and worst status + if fix_theta or multistart or partial_fix_mode: + return obj_value, theta_estimates, worst_status + + # Return theta estimates as a pandas Series + theta_estimates = pd.Series(theta_estimates) + + # Extract return values if requested + # Assumes the model components are named the same in each block, and are pyo.Vars. + if return_values is not None and len(return_values) > 0: + var_values = [] + # In the scenario blocks structure, exp_scenarios is an IndexedBlock + exp_blocks = self.ef_instance.exp_scenarios.values() + # Loop over each experiment block and extract requested variable values + for exp_i in exp_blocks: + # In each block, extract requested variables + vals = {} + for var in return_values: + # Find the variable in the block + exp_i_var = exp_i.find_component(str(var)) + # Check if variable exists in the block + if exp_i_var is None: + continue + # Extract value(s) from variable + if type(exp_i_var) == ContinuousSet: + temp = list(exp_i_var) + else: + temp = [pyo.value(_) for _ in exp_i_var.values()] + if len(temp) == 1: + vals[var] = temp[0] + else: + vals[var] = temp + # Only append if vals is not empty + if len(vals) > 0: + var_values.append(vals) + # Convert to DataFrame + var_values = pd.DataFrame(var_values) + + if return_values is not None and len(return_values) > 0: + return obj_value, theta_estimates, var_values + else: + return obj_value, theta_estimates + + # Removed old _Q_opt function def _cov_at_theta(self, method, solver, step): """ @@ -1181,14 +2112,20 @@ def _cov_at_theta(self, method, solver, step): cov : pd.DataFrame Covariance matrix of the estimated parameters """ + if hasattr(self.ef_instance, "exp_scenarios"): + ref_model = self.ef_instance.exp_scenarios[0] + else: + ref_model = self.ef_instance + if method == CovarianceMethod.reduced_hessian.value: # compute the inverse reduced hessian to be used # in the "reduced_hessian" method - # parmest makes the fitted parameters stage 1 variables + # retrieve the independent variables (i.e., estimated parameters) ind_vars = [] - for nd_name, Var, sol_val in ef_nonants(self.ef_instance): - ind_vars.append(Var) - # calculate the reduced hessian + for name in self.ef_instance._parmest_theta_names: + var = self.ef_instance.parmest_theta[name] + ind_vars.append(var) + solve_result, inv_red_hes = ( inverse_reduced_hessian.inv_reduced_hessian_barrier( self.ef_instance, @@ -1199,6 +2136,43 @@ def _cov_at_theta(self, method, solver, step): ) self.inv_red_hes = inv_red_hes + else: + # if not using the 'reduced_hessian' method, calculate the sum of squared errors + # using 'finite_difference' method or 'automatic_differentiation_kaug' + sse_vals = [] + for experiment in self.exp_list: + model = _get_labeled_model(experiment) + + # fix the value of the unknown parameters to the estimated values + for param in model.unknown_parameters: + param.fix(self.estimated_theta[param.name]) + + # re-solve the model with the estimated parameters + results = pyo.SolverFactory(solver).solve(model, tee=self.tee) + assert_optimal_termination(results) + + # choose and evaluate the sum of squared errors expression + if self.obj_function == ObjectiveType.SSE: + sse_expr = SSE(model) + elif self.obj_function == ObjectiveType.SSE_weighted: + sse_expr = SSE_weighted(model) + else: + raise ValueError( + f"Invalid objective function for covariance calculation. " + f"The covariance matrix can only be calculated using the built-in " + f"objective functions: {[e.value for e in ObjectiveType]}. Supply " + f"the Estimator object one of these built-in objectives and " + f"re-run the code." + ) + + # evaluate the numerical SSE and store it + sse_val = pyo.value(sse_expr) + sse_vals.append(sse_val) + + sse = sum(sse_vals) + logger.info( + f"The sum of squared errors at the estimated parameter(s) is: {sse}" + ) # Number of data points considered n = self.number_exp @@ -1206,42 +2180,6 @@ def _cov_at_theta(self, method, solver, step): # Extract the number of fitted parameters l = len(self.estimated_theta) - # calculate the sum of squared errors at the estimated parameter values - sse_vals = [] - for experiment in self.exp_list: - model = _get_labeled_model(experiment) - - # fix the value of the unknown parameters to the estimated values - for param in model.unknown_parameters: - param.fix(self.estimated_theta[param.name]) - - # re-solve the model with the estimated parameters - results = pyo.SolverFactory(solver).solve(model, tee=self.tee) - assert_optimal_termination(results) - - # choose and evaluate the sum of squared errors expression - if self.obj_function == ObjectiveType.SSE: - sse_expr = SSE(model) - elif self.obj_function == ObjectiveType.SSE_weighted: - sse_expr = SSE_weighted(model) - else: - raise ValueError( - f"Invalid objective function for covariance calculation. " - f"The covariance matrix can only be calculated using the built-in " - f"objective functions: {[e.value for e in ObjectiveType]}. Supply " - f"the Estimator object one of these built-in objectives and " - f"re-run the code." - ) - - # evaluate the numerical SSE and store it - sse_val = pyo.value(sse_expr) - sse_vals.append(sse_val) - - sse = sum(sse_vals) - logger.info( - f"The sum of squared errors at the estimated parameter(s) is: {sse}" - ) - """Calculate covariance assuming experimental observation errors are independent and follow a Gaussian distribution with constant variance. @@ -1262,13 +2200,19 @@ def _cov_at_theta(self, method, solver, step): ) # check if the user specified 'SSE' or 'SSE_weighted' as the objective function + cov_prior_FIM = None + cov_regularization_weight = None + if self.regularization == RegularizationType.L2: + cov_prior_FIM = self.prior_FIM + cov_regularization_weight = self.regularization_weight + if self.obj_function == ObjectiveType.SSE: # check if the user defined the 'measurement_error' attribute - if hasattr(model, "measurement_error"): + if hasattr(ref_model, "measurement_error"): # get the measurement errors meas_error = [ - model.measurement_error[y_hat] - for y_hat, y in model.experiment_outputs.items() + ref_model.measurement_error[y_hat] + for y_hat, y in ref_model.experiment_outputs.items() ] # check if the user supplied the values of the measurement errors @@ -1298,6 +2242,8 @@ def _cov_at_theta(self, method, solver, step): method, obj_function=self.covariance_objective, theta_vals=self.estimated_theta, + prior_FIM=cov_prior_FIM, + regularization_weight=cov_regularization_weight, solver=solver, step=step, tee=self.tee, @@ -1328,7 +2274,10 @@ def _cov_at_theta(self, method, solver, step): solver=solver, step=step, tee=self.tee, + prior_FIM=cov_prior_FIM, + regularization_weight=cov_regularization_weight, ) + else: raise ValueError( "One or more values of the measurement errors have " @@ -1340,10 +2289,10 @@ def _cov_at_theta(self, method, solver, step): ) elif self.obj_function == ObjectiveType.SSE_weighted: # check if the user defined the 'measurement_error' attribute - if hasattr(model, "measurement_error"): + if hasattr(ref_model, "measurement_error"): meas_error = [ - model.measurement_error[y_hat] - for y_hat, y in model.experiment_outputs.items() + ref_model.measurement_error[y_hat] + for y_hat, y in ref_model.experiment_outputs.items() ] # check if the user supplied the values for the measurement errors @@ -1366,6 +2315,8 @@ def _cov_at_theta(self, method, solver, step): method, obj_function=self.covariance_objective, theta_vals=self.estimated_theta, + prior_FIM=cov_prior_FIM, + regularization_weight=cov_regularization_weight, step=step, solver=solver, tee=self.tee, @@ -1380,200 +2331,16 @@ def _cov_at_theta(self, method, solver, step): raise AttributeError( 'Experiment model does not have suffix "measurement_error".' ) - - return cov - - def _Q_at_theta(self, thetavals, initialize_parmest_model=False): - """ - Return the objective function value with fixed theta values. - - Parameters - ---------- - thetavals: dict - A dictionary of theta values. - - initialize_parmest_model: boolean - If True: Solve square problem instance, build extensive form of the model for - parameter estimation, and set flag model_initialized to True. Default is False. - - Returns - ------- - objectiveval: float - The objective function value. - thetavals: dict - A dictionary of all values for theta that were input. - solvertermination: Pyomo TerminationCondition - Tries to return the "worst" solver status across the scenarios. - pyo.TerminationCondition.optimal is the best and - pyo.TerminationCondition.infeasible is the worst. - """ - - optimizer = pyo.SolverFactory('ipopt') - - if len(thetavals) > 0: - dummy_cb = { - "callback": self._instance_creation_callback, - "ThetaVals": thetavals, - "theta_names": self._return_theta_names(), - "cb_data": None, - } else: - dummy_cb = { - "callback": self._instance_creation_callback, - "theta_names": self._return_theta_names(), - "cb_data": None, - } - - if self.diagnostic_mode: - if len(thetavals) > 0: - print(' Compute objective at theta = ', str(thetavals)) - else: - print(' Compute objective at initial theta') - - # start block of code to deal with models with no constraints - # (ipopt will crash or complain on such problems without special care) - instance = _experiment_instance_creation_callback("FOO0", None, dummy_cb) - try: # deal with special problems so Ipopt will not crash - first = next(instance.component_objects(pyo.Constraint, active=True)) - active_constraints = True - except: - active_constraints = False - # end block of code to deal with models with no constraints - - WorstStatus = pyo.TerminationCondition.optimal - totobj = 0 - scenario_numbers = list(range(len(self.exp_list))) - if initialize_parmest_model: - # create dictionary to store pyomo model instances (scenarios) - scen_dict = dict() - - for snum in scenario_numbers: - sname = "scenario_NODE" + str(snum) - instance = _experiment_instance_creation_callback(sname, None, dummy_cb) - model_theta_names = self._expand_indexed_unknowns(instance) - - if initialize_parmest_model: - # list to store fitted parameter names that will be unfixed - # after initialization - theta_init_vals = [] - # use appropriate theta_names member - theta_ref = model_theta_names - - for i, theta in enumerate(theta_ref): - # Use parser in ComponentUID to locate the component - var_cuid = ComponentUID(theta) - var_validate = var_cuid.find_component_on(instance) - if var_validate is None: - logger.warning( - "theta_name %s was not found on the model", (theta) - ) - else: - try: - if len(thetavals) == 0: - var_validate.fix() - else: - var_validate.fix(thetavals[theta]) - theta_init_vals.append(var_validate) - except: - logger.warning( - 'Unable to fix model parameter value for %s (not a Pyomo model Var)', - (theta), - ) - - if active_constraints: - if self.diagnostic_mode: - print(' Experiment = ', snum) - print(' First solve with special diagnostics wrapper') - status_obj, solved, iters, time, regu = ( - utils.ipopt_solve_with_stats( - instance, optimizer, max_iter=500, max_cpu_time=120 - ) - ) - print( - " status_obj, solved, iters, time, regularization_stat = ", - str(status_obj), - str(solved), - str(iters), - str(time), - str(regu), - ) - - results = optimizer.solve(instance) - if self.diagnostic_mode: - print( - 'standard solve solver termination condition=', - str(results.solver.termination_condition), - ) - - if ( - results.solver.termination_condition - != pyo.TerminationCondition.optimal - ): - # DLW: Aug2018: not distinguishing "middlish" conditions - if WorstStatus != pyo.TerminationCondition.infeasible: - WorstStatus = results.solver.termination_condition - if initialize_parmest_model: - if self.diagnostic_mode: - print( - "Scenario {:d} infeasible with initialized parameter values".format( - snum - ) - ) - else: - if initialize_parmest_model: - if self.diagnostic_mode: - print( - "Scenario {:d} initialization successful with initial parameter values".format( - snum - ) - ) - if initialize_parmest_model: - # unfix parameters after initialization - for theta in theta_init_vals: - theta.unfix() - scen_dict[sname] = instance - else: - if initialize_parmest_model: - # unfix parameters after initialization - for theta in theta_init_vals: - theta.unfix() - scen_dict[sname] = instance - - objobject = getattr(instance, self._second_stage_cost_exp) - objval = pyo.value(objobject) - totobj += objval - - retval = totobj / len(scenario_numbers) # -1?? - if initialize_parmest_model and not hasattr(self, 'ef_instance'): - # create extensive form of the model using scenario dictionary - if len(scen_dict) > 0: - for scen in scen_dict.values(): - scen._mpisppy_probability = 1 / len(scen_dict) - - if use_mpisppy: - EF_instance = sputils._create_EF_from_scen_dict( - scen_dict, - EF_name="_Q_at_theta", - # suppress_warnings=True - ) - else: - EF_instance = local_ef._create_EF_from_scen_dict( - scen_dict, EF_name="_Q_at_theta", nonant_for_fixed_vars=True - ) - - self.ef_instance = EF_instance - # set self.model_initialized flag to True to skip extensive form model - # creation using theta_est() - self.model_initialized = True - - # return initialized theta values - if len(thetavals) == 0: - # use appropriate theta_names member - theta_ref = self._return_theta_names() - for i, theta in enumerate(theta_ref): - thetavals[theta] = theta_init_vals[i]() + raise ValueError( + f"Invalid objective function for covariance calculation. " + f"The covariance matrix can only be calculated using the built-in " + f"objective functions: {[e.value for e in ObjectiveType]}. Supply " + f"the Estimator object one of these built-in objectives and " + f"re-run the code." + ) - return retval, thetavals, WorstStatus + return cov def _get_sample_list(self, samplesize, num_samples, replacement=True): samplelist = list() @@ -1610,6 +2377,8 @@ def _get_sample_list(self, samplesize, num_samples, replacement=True): return samplelist + # @Reviewers: Currently regularization chosen with objective at Estimator initialization, + # Would it be preferable to have regularization choice as an argument in the theta_est function instead? def theta_est( self, solver="ef_ipopt", return_values=[], calc_cov=NOTSET, cov_n=NOTSET ): @@ -1671,13 +2440,7 @@ def theta_est( solver=solver, return_values=return_values ) - return self._Q_opt( - solver=solver, - return_values=return_values, - bootlist=None, - calc_cov=calc_cov, - cov_n=cov_n, - ) + return self._Q_opt(solver=solver, return_values=return_values, bootlist=None) def cov_est(self, method="finite_difference", solver="ipopt", step=1e-3): """ @@ -1944,9 +2707,398 @@ def leaveNout_bootstrap_test( return results + def theta_est_multistart( + self, + n_restarts=20, + multistart_sampling_method="uniform_random", + user_provided_df=None, + seed=None, + save_results=False, + file_name="multistart_results.csv", + ): + """ + Run multistart parameter estimation and aggregate per-start results. + + A canonical starts/results table is created first, then each start is + solved (potentially in parallel with ``ParallelTaskManager``), and the + output table is populated with objective values, solver terminations, + solve times, and converged theta values. + + Parameters + ---------- + n_restarts : int, optional + Number of starts for sampled methods. Ignored when + ``user_provided_df`` is provided. + multistart_sampling_method : str, optional + Sampling method for generated starts. + user_provided_df : DataFrame, optional + User-provided starts. If provided, these rows define the restart + set directly. + seed : int, optional + Seed used by sampling methods. + save_results : bool, optional + If True, write the full results DataFrame to ``file_name``. + file_name : str, optional + Output CSV path used when ``save_results`` is True. + + Returns + ------- + tuple + ``(results_df, best_theta, best_obj)``, where: + - ``results_df`` contains one row per start plus converged metadata + - ``best_theta`` is the selected best feasible theta dictionary or + ``None`` if no finite objective exists + - ``best_obj`` is the selected objective value or ``np.nan`` + + Notes + ----- + Best-run selection prioritizes acceptable solver terminations + (optimal/locallyOptimal/globallyOptimal) and then minimizes objective. + If no acceptable statuses exist, finite-objective rows are considered. + """ + if self.pest_deprecated is not None: + raise RuntimeError( + "Multistart is not supported in the deprecated parmest interface." + ) + if user_provided_df is None: + if not isinstance(n_restarts, int): + raise TypeError("n_restarts must be an integer.") + if n_restarts <= 0: + raise ValueError("n_restarts must be greater than zero.") + + n_restarts_for_generation = None if user_provided_df is not None else n_restarts + results_df = self._generate_initial_theta( + seed=seed, + n_restarts=n_restarts_for_generation, + multistart_sampling_method=multistart_sampling_method, + user_provided_df=user_provided_df, + experiment_number=0, + ) + theta_names = [ + c + for c in results_df.columns + if not c.startswith("converged_") + and c not in {"final objective", "solver termination", "solve_time"} + ] + n_restarts = results_df.shape[0] + + # Convert each row to (row_index, theta_dict) + tasks = [] + for i in range(results_df.shape[0]): + Theta = {name: float(results_df.iloc[i][name]) for name in theta_names} + tasks.append((i, Theta)) + + task_mgr = utils.ParallelTaskManager(len(tasks)) + local_tasks = task_mgr.global_to_local_data(tasks) + + # Solve in parallel + local_results = [] + for i, Theta in local_tasks: + import time + + t0 = time.time() + try: + final_obj, theta_hat, worst = self._Q_opt( + theta_vals=Theta, multistart=True + ) + solve_time = time.time() - t0 + local_results.append((i, final_obj, str(worst), solve_time, theta_hat)) + except Exception as exc: + solve_time = time.time() - t0 + local_results.append( + ( + i, + np.nan, + f"exception(start={i}, sampler={multistart_sampling_method}): {exc}", + solve_time, + None, + ) + ) + + global_results = task_mgr.allgather_global_data(local_results) + + # Fill results_df + for i, final_obj, term, solve_time, theta_hat in global_results: + results_df.at[i, "final objective"] = final_obj + results_df.at[i, "solver termination"] = term + results_df.at[i, "solve_time"] = solve_time + + if theta_hat is not None: + for name in theta_names: + if name in theta_hat: + results_df.at[i, f"converged_{name}"] = float(theta_hat[name]) + + # Best solution: + # prioritize starts with acceptable solver terminations, then minimum objective. + acceptable_terms = { + str(pyo.TerminationCondition.optimal), + str(pyo.TerminationCondition.locallyOptimal), + str(pyo.TerminationCondition.globallyOptimal), + } + finite_obj_mask = np.isfinite( + results_df["final objective"].to_numpy(dtype=float) + ) + acceptable_mask = results_df["solver termination"].isin(acceptable_terms) + ranked = results_df[finite_obj_mask & acceptable_mask] + if ranked.empty: + ranked = results_df[finite_obj_mask] + + if ranked.empty: + best_theta = None + best_obj = np.nan + else: + best_idx = ranked["final objective"].astype(float).idxmin() + best_obj = float(results_df.loc[best_idx, "final objective"]) + best_theta = { + name: float(results_df.loc[best_idx, f"converged_{name}"]) + for name in theta_names + } + + if save_results: + results_df.to_csv(file_name, index=False) + + return results_df, best_theta, best_obj + + def _build_profile_grid(self, profiled_theta, grid, n_grid, theta_hat): + """Build sorted profile grid for a single profiled parameter.""" + if not isinstance(profiled_theta, str): + raise TypeError("profiled_theta must be a string.") + if grid is not None and not isinstance( + grid, (list, tuple, np.ndarray, pd.Series) + ): + raise TypeError("grid must be a sequence of numeric values or None.") + if not isinstance(n_grid, int): + raise TypeError("n_grid must be an integer.") + if n_grid < 2: + raise ValueError("n_grid must be at least 2.") + if not isinstance(theta_hat, dict): + raise TypeError("theta_hat must be a dictionary.") + if profiled_theta not in theta_hat: + raise ValueError(f"theta_hat does not include '{profiled_theta}'.") + + template_model = self._create_parmest_model(0) + theta_var = template_model.find_component(profiled_theta) + if theta_var is None: + raise ValueError(f"Unknown theta name '{profiled_theta}'.") + + theta_hat_val = float(theta_hat[profiled_theta]) + if grid is None: + if theta_var.lb is None or theta_var.ub is None: + raise ValueError( + f"Cannot auto-build grid for '{profiled_theta}' without lower and upper bounds." + ) + values = np.linspace(float(theta_var.lb), float(theta_var.ub), n_grid) + else: + values = np.asarray(list(grid), dtype=float) + if values.size == 0: + raise ValueError("Provided grid must contain at least one value.") + + values = np.append(values, theta_hat_val) + values = np.unique(np.round(values.astype(float), decimals=14)) + values.sort() + return values + + def _order_grid_for_continuation(self, grid_values, center): + """Return center-out ordering for continuation warm starts.""" + ordered = sorted( + [float(v) for v in grid_values], key=lambda v: (abs(v - center), v) + ) + return ordered + + def profile_likelihood( + self, + profiled_theta, + grid=None, + n_grid=21, + theta_hat=None, + obj_hat=None, + use_multistart_for_baseline=False, + baseline_multistart_kwargs=None, + solver="ef_ipopt", + warmstart="neighbor", + max_consecutive_failures=None, + return_theta_paths=True, + seed=None, + ): + """ + Compute one-dimensional profile likelihood curves by fixing one parameter + at grid values and re-optimizing all other parameters. + """ + if self.pest_deprecated is not None: + raise RuntimeError( + "Profile likelihood is not supported in the deprecated parmest interface." + ) + if isinstance(profiled_theta, str): + profiled_theta_list = [profiled_theta] + elif isinstance(profiled_theta, (list, tuple)): + profiled_theta_list = list(profiled_theta) + else: + raise TypeError( + "profiled_theta must be a string or a list/tuple of strings." + ) + if len(profiled_theta_list) == 0: + raise ValueError("At least one profiled theta must be provided.") + if not all(isinstance(p, str) for p in profiled_theta_list): + raise TypeError("Each profiled theta name must be a string.") + if warmstart not in ("neighbor", "none"): + raise ValueError("warmstart must be either 'neighbor' or 'none'.") + if max_consecutive_failures is not None: + if not isinstance(max_consecutive_failures, int): + raise TypeError("max_consecutive_failures must be an integer or None.") + if max_consecutive_failures <= 0: + raise ValueError("max_consecutive_failures must be greater than zero.") + if not isinstance(return_theta_paths, bool): + raise TypeError("return_theta_paths must be a bool.") + if seed is not None and not isinstance(seed, int): + raise TypeError("seed must be an integer or None.") + + theta_names = self._expand_indexed_unknowns(self._create_parmest_model(0)) + unknown_requested = set(profiled_theta_list).difference(theta_names) + if unknown_requested: + raise ValueError( + f"Unknown profile theta name(s): {sorted(unknown_requested)}. " + f"Known names: {theta_names}." + ) + + if seed is not None: + np.random.seed(seed) + + if theta_hat is None or obj_hat is None: + if use_multistart_for_baseline: + ms_kwargs = dict(baseline_multistart_kwargs or {}) + _, best_theta, best_obj = self.theta_est_multistart(**ms_kwargs) + if best_theta is None or not np.isfinite(best_obj): + raise RuntimeError( + "Failed to compute baseline from multistart: no feasible solution found." + ) + theta_hat = best_theta + obj_hat = best_obj + else: + obj_hat, theta_hat_series = self.theta_est(solver=solver) + theta_hat = theta_hat_series.to_dict() + + theta_hat = {k: float(v) for k, v in theta_hat.items()} + obj_hat = float(obj_hat) + + rows = [] + acceptable_terms = { + str(pyo.TerminationCondition.optimal), + str(pyo.TerminationCondition.locallyOptimal), + str(pyo.TerminationCondition.globallyOptimal), + } + started_at = time.time() + + for pname in profiled_theta_list: + if isinstance(grid, dict): + grid_spec = grid.get(pname, None) + else: + grid_spec = grid + grid_values = self._build_profile_grid( + profiled_theta=pname, grid=grid_spec, n_grid=n_grid, theta_hat=theta_hat + ) + + if warmstart == "neighbor": + ordered_grid = self._order_grid_for_continuation( + grid_values, center=theta_hat[pname] + ) + else: + ordered_grid = [float(v) for v in grid_values] + + previous_theta = dict(theta_hat) + consecutive_failures = 0 + + for g in ordered_grid: + init_theta = dict(theta_hat) + if warmstart == "neighbor": + init_theta.update(previous_theta) + init_theta[pname] = float(g) + + t0 = time.time() + status = "" + success = False + obj_val = np.nan + theta_conv = None + try: + obj_val, theta_conv, term = self._Q_opt( + theta_vals=init_theta, + fixed_theta_values={pname: float(g)}, + solver=solver, + ) + status = str(term) + success = status in acceptable_terms and np.isfinite(obj_val) + if success: + consecutive_failures = 0 + if warmstart == "neighbor": + previous_theta = dict(theta_conv) + else: + consecutive_failures += 1 + except Exception as exc: + status = f"exception: {exc}" + consecutive_failures += 1 + + row = { + "profiled_theta": pname, + "theta_value": float(g), + "obj": float(obj_val) if np.isfinite(obj_val) else np.nan, + "status": status, + "success": bool(success), + "solve_time": float(time.time() - t0), + } + if return_theta_paths and isinstance(theta_conv, dict): + for tname, tval in theta_conv.items(): + row[f"theta__{tname}"] = float(tval) + rows.append(row) + + if ( + max_consecutive_failures is not None + and consecutive_failures >= max_consecutive_failures + ): + break + + profiles = pd.DataFrame(rows) + if profiles.empty: + profiles = pd.DataFrame( + columns=[ + "profiled_theta", + "theta_value", + "obj", + "status", + "success", + "solve_time", + "delta_obj", + "lr_stat", + ] + ) + else: + profiles = profiles.sort_values( + by=["profiled_theta", "theta_value"], kind="stable" + ).reset_index(drop=True) + profiles["delta_obj"] = profiles["obj"] - obj_hat + profiles["lr_stat"] = 2.0 * profiles["delta_obj"] + + return { + "profiles": profiles, + "baseline": { + "theta_hat": theta_hat, + "obj_hat": obj_hat, + "solver": solver, + "used_multistart": bool(use_multistart_for_baseline), + }, + "metadata": { + "grid_strategy": "user_provided" if grid is not None else "auto_bounds", + "warmstart": warmstart, + "seed": seed, + "profiled_theta": list(profiled_theta_list), + "started_at_epoch": float(started_at), + "finished_at_epoch": float(time.time()), + }, + } + + # Updated version that uses _Q_opt def objective_at_theta(self, theta_values=None, initialize_parmest_model=False): """ - Objective value for each theta + Objective value for each theta, solving extensive form problem with + fixed theta values. Parameters ---------- @@ -1958,7 +3110,6 @@ def objective_at_theta(self, theta_values=None, initialize_parmest_model=False): of the model for parameter estimation, and set flag model_initialized to True. Default is False. - Returns ------- obj_at_theta: pd.DataFrame @@ -1973,65 +3124,73 @@ def objective_at_theta(self, theta_values=None, initialize_parmest_model=False): initialize_parmest_model=initialize_parmest_model, ) - if len(self.estimator_theta_names) == 0: - pass # skip assertion if model has no fitted parameters - else: - # create a local instance of the pyomo model to access model variables and parameters - model_temp = self._create_parmest_model(0) - model_theta_list = self._expand_indexed_unknowns(model_temp) - - # if self.estimator_theta_names is not the same as temp model_theta_list, - # create self.theta_names_updated - if set(self.estimator_theta_names) == set(model_theta_list) and len( - self.estimator_theta_names - ) == len(set(model_theta_list)): - pass - else: - self.theta_names_updated = model_theta_list + if initialize_parmest_model: + # Print deprecation warning, that this option will be removed in + # future releases. + deprecation_warning( + "The `initialize_parmest_model` option in `objective_at_theta()` is " + "deprecated and will be removed in future releases. Please ensure the" + "model is initialized within the Experiment class definition.", + version="6.9.5", + ) if theta_values is None: all_thetas = {} # dictionary to store fitted variables # use appropriate theta names member - theta_names = model_theta_list + # Get theta names from fresh parmest model, assuming this can be called + # directly after creating Estimator. + theta_names = self._expand_indexed_unknowns(self._create_parmest_model(0)) else: assert isinstance(theta_values, pd.DataFrame) # for parallel code we need to use lists and dicts in the loop theta_names = theta_values.columns # # check if theta_names are in model - for theta in list(theta_names): - theta_temp = theta.replace("'", "") # cleaning quotes from theta_names - assert theta_temp in [ - t.replace("'", "") for t in model_theta_list - ], "Theta name {} in 'theta_values' not in 'theta_names' {}".format( - theta_temp, model_theta_list + # Clean names, ignore quotes, and compare sets + clean_provided = [t.replace("'", "") for t in theta_names] + if len(clean_provided) != len(set(clean_provided)): + raise ValueError( + f"Duplicate theta names are not allowed: {clean_provided}" ) + clean_expected = [ + t.replace("'", "") + for t in self._expand_indexed_unknowns(self._create_parmest_model(0)) + ] + # If they do not match, raise error + if (len(clean_provided) != len(clean_expected)) or ( + set(clean_provided) != set(clean_expected) + ): + raise ValueError( + f"Provided theta values {clean_provided} do not match expected theta names {clean_expected}." + ) + # Rename columns using cleaned names + if list(clean_provided) != list(theta_names): + theta_values = theta_values.copy() + theta_values.columns = clean_provided - assert len(list(theta_names)) == len(model_theta_list) - + # Convert to list of dicts for parallel processing all_thetas = theta_values.to_dict('records') + # Initialize task manager + num_tasks = len(all_thetas) if all_thetas else 1 + task_mgr = utils.ParallelTaskManager(num_tasks) + + # Use local theta values for each task if all_thetas is provided, else empty list if all_thetas: - task_mgr = utils.ParallelTaskManager(len(all_thetas)) local_thetas = task_mgr.global_to_local_data(all_thetas) - else: - if initialize_parmest_model: - task_mgr = utils.ParallelTaskManager( - 1 - ) # initialization performed using just 1 set of theta values + elif initialize_parmest_model: + local_thetas = [] + # walk over the mesh, return objective function all_obj = list() if len(all_thetas) > 0: for Theta in local_thetas: - obj, thetvals, worststatus = self._Q_at_theta( - Theta, initialize_parmest_model=initialize_parmest_model + obj, thetvals, worststatus = self._Q_opt( + theta_vals=Theta, fix_theta=True ) if worststatus != pyo.TerminationCondition.infeasible: all_obj.append(list(Theta.values()) + [obj]) - # DLW, Aug2018: should we also store the worst solver status? else: - obj, thetvals, worststatus = self._Q_at_theta( - thetavals={}, initialize_parmest_model=initialize_parmest_model - ) + obj, thetvals, worststatus = self._Q_opt(theta_vals=None, fix_theta=True) if worststatus != pyo.TerminationCondition.infeasible: all_obj.append(list(thetvals.values()) + [obj]) diff --git a/pyomo/contrib/parmest/tests/test_examples.py b/pyomo/contrib/parmest/tests/test_examples.py index 94a83d1f058..272f2944e82 100644 --- a/pyomo/contrib/parmest/tests/test_examples.py +++ b/pyomo/contrib/parmest/tests/test_examples.py @@ -8,6 +8,7 @@ # ____________________________________________________________________________________ import pyomo.common.unittest as unittest +import math import pyomo.contrib.parmest.parmest as parmest from pyomo.contrib.parmest.graphics import matplotlib_available, seaborn_available from pyomo.contrib.pynumero.asl import AmplInterface @@ -54,6 +55,25 @@ def test_likelihood_ratio_example(self): likelihood_ratio_example.main() + def test_regularization_example(self): + from pyomo.contrib.parmest.examples.rooney_biegler import regularization_example + + results = regularization_example.main() + # Keep this as a lightweight contract test: example must return both + # regularization modes with finite outputs. + self.assertIn("L1", results) + self.assertIn("L2", results) + + l1_obj, l1_theta, _ = results["L1"] + l2_obj, l2_theta, _ = results["L2"] + + self.assertTrue(math.isfinite(float(l1_obj))) + self.assertTrue(math.isfinite(float(l2_obj))) + self.assertTrue(math.isfinite(float(l1_theta["rate_constant"]))) + self.assertTrue(math.isfinite(float(l2_theta["rate_constant"]))) + self.assertGreaterEqual(float(l1_theta["rate_constant"]), 0.0) + self.assertLessEqual(float(l1_theta["rate_constant"]), 5.0) + @unittest.skipUnless(pynumero_ASL_available, "test requires libpynumero_ASL") @unittest.skipUnless(ipopt_available, "The 'ipopt' solver is not available") diff --git a/pyomo/contrib/parmest/tests/test_graphics.py b/pyomo/contrib/parmest/tests/test_graphics.py index 42c0b2a9f1f..e9a7b1c75fd 100644 --- a/pyomo/contrib/parmest/tests/test_graphics.py +++ b/pyomo/contrib/parmest/tests/test_graphics.py @@ -61,6 +61,28 @@ def test_grouped_boxplot(self): def test_grouped_violinplot(self): graphics.grouped_violinplot(self.A, self.B) + def test_profile_plot_smoke_single_parameter(self): + prof = pd.DataFrame( + { + "profiled_theta": ["theta"] * 3, + "theta_value": [0.8, 1.0, 1.2], + "lr_stat": [3.0, 0.0, 4.0], + "success": [True, True, False], + } + ) + graphics.profile_likelihood_plot({"profiles": prof}, alpha=0.95) + + def test_profile_plot_smoke_multi_parameter(self): + prof = pd.DataFrame( + { + "profiled_theta": ["theta_a", "theta_a", "theta_b", "theta_b"], + "theta_value": [1.0, 2.0, -1.0, 0.0], + "lr_stat": [2.0, 0.0, 1.0, 0.0], + "success": [True, True, True, True], + } + ) + graphics.profile_likelihood_plot({"profiles": prof}, alpha=0.9) + if __name__ == '__main__': unittest.main() diff --git a/pyomo/contrib/parmest/tests/test_parmest.py b/pyomo/contrib/parmest/tests/test_parmest.py index dfeab769e71..d8a2fea003b 100644 --- a/pyomo/contrib/parmest/tests/test_parmest.py +++ b/pyomo/contrib/parmest/tests/test_parmest.py @@ -11,7 +11,6 @@ import os import subprocess from itertools import product - from pyomo.common.unittest import pytest from parameterized import parameterized, parameterized_class import pyomo.common.unittest as unittest @@ -30,6 +29,7 @@ pynumero_ASL_available = AmplInterface.available() testdir = this_file_dir() +# TESTS HERE WILL BE MODIFIED FOR _Q_OPT_BLOCKS LATER # Set the global seed for random number generation in tests _RANDOM_SEED_FOR_TESTING = 524 @@ -508,40 +508,6 @@ def test_parallel_parmest(self): retcode = subprocess.call(rlist) self.assertEqual(retcode, 0) - @unittest.skipIf(not pynumero_ASL_available, "pynumero_ASL is not available") - def test_theta_est_cov(self): - objval, thetavals, cov = self.pest.theta_est(calc_cov=True, cov_n=6) - - self.assertAlmostEqual(objval, 4.3317112, places=2) - self.assertAlmostEqual( - thetavals["asymptote"], 19.1426, places=2 - ) # 19.1426 from the paper - self.assertAlmostEqual( - thetavals["rate_constant"], 0.5311, places=2 - ) # 0.5311 from the paper - - # Covariance matrix - self.assertAlmostEqual( - cov["asymptote"]["asymptote"], 6.155892, places=2 - ) # 6.22864 from paper - self.assertAlmostEqual( - cov["asymptote"]["rate_constant"], -0.425232, places=2 - ) # -0.4322 from paper - self.assertAlmostEqual( - cov["rate_constant"]["asymptote"], -0.425232, places=2 - ) # -0.4322 from paper - self.assertAlmostEqual( - cov["rate_constant"]["rate_constant"], 0.040571, places=2 - ) # 0.04124 from paper - - """ Why does the covariance matrix from parmest not match the paper? Parmest is - calculating the exact reduced Hessian. The paper (Rooney and Bielger, 2001) likely - employed the first order approximation common for nonlinear regression. The paper - values were verified with Scipy, which uses the same first order approximation. - The formula used in parmest was verified against equations (7-5-15) and (7-5-16) in - "Nonlinear Parameter Estimation", Y. Bard, 1974. - """ - def test_cov_scipy_least_squares_comparison(self): """ Scipy results differ in the 3rd decimal place from the paper. It is possible @@ -655,20 +621,27 @@ def setUp(self): columns=["hour", "y"], ) + # Updated models to use Vars for experiment output, and Constraints def rooney_biegler_params(data): model = pyo.ConcreteModel() model.asymptote = pyo.Param(initialize=15, mutable=True) model.rate_constant = pyo.Param(initialize=0.5, mutable=True) - model.hour = pyo.Param(within=pyo.PositiveReals, mutable=True) - model.y = pyo.Param(within=pyo.PositiveReals, mutable=True) + # Add the experiment inputs + model.h = pyo.Var(initialize=data["hour"].iloc[0], bounds=(0, 10)) - def response_rule(m, h): - expr = m.asymptote * (1 - pyo.exp(-m.rate_constant * h)) - return expr + # Fix the experiment inputs + model.h.fix() - model.response_function = pyo.Expression(data.hour, rule=response_rule) + # Add experiment outputs + model.y = pyo.Var(initialize=data['y'].iloc[0], within=pyo.PositiveReals) + + # Define the model equations + def response_rule(m): + return m.y == m.asymptote * (1 - pyo.exp(-m.rate_constant * m.h)) + + model.response_con = pyo.Constraint(rule=response_rule) return model @@ -683,14 +656,14 @@ def label_model(self): m = self.model m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) - m.experiment_outputs.update( - [(m.hour, self.data["hour"]), (m.y, self.data["y"])] - ) + m.experiment_outputs.update([(m.y, self.data["y"])]) m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) m.unknown_parameters.update( (k, pyo.ComponentUID(k)) for k in [m.asymptote, m.rate_constant] ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) rooney_biegler_params_exp_list = [] for i in range(self.data.shape[0]): @@ -701,24 +674,30 @@ def label_model(self): def rooney_biegler_indexed_params(data): model = pyo.ConcreteModel() + # Define the indexed parameters model.param_names = pyo.Set(initialize=["asymptote", "rate_constant"]) model.theta = pyo.Param( model.param_names, initialize={"asymptote": 15, "rate_constant": 0.5}, mutable=True, ) + # Add the experiment inputs + model.h = pyo.Var(initialize=data["hour"].iloc[0], bounds=(0, 10)) - model.hour = pyo.Param(within=pyo.PositiveReals, mutable=True) - model.y = pyo.Param(within=pyo.PositiveReals, mutable=True) + # Fix the experiment inputs + model.h.fix() - def response_rule(m, h): - expr = m.theta["asymptote"] * ( - 1 - pyo.exp(-m.theta["rate_constant"] * h) - ) - return expr + # Add experiment outputs + model.y = pyo.Var(initialize=data['y'].iloc[0], within=pyo.PositiveReals) - model.response_function = pyo.Expression(data.hour, rule=response_rule) + # Define the model equations + def response_rule(m): + return m.y == m.theta["asymptote"] * ( + 1 - pyo.exp(-m.theta["rate_constant"] * m.h) + ) + # Add the model equations to the model + model.response_con = pyo.Constraint(rule=response_rule) return model class RooneyBieglerExperimentIndexedParams(RooneyBieglerExperiment): @@ -732,13 +711,14 @@ def label_model(self): m = self.model m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) - m.experiment_outputs.update( - [(m.hour, self.data["hour"]), (m.y, self.data["y"])] - ) + m.experiment_outputs.update([(m.y, self.data["y"])]) m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) m.unknown_parameters.update((k, pyo.ComponentUID(k)) for k in [m.theta]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + rooney_biegler_indexed_params_exp_list = [] for i in range(self.data.shape[0]): rooney_biegler_indexed_params_exp_list.append( @@ -753,14 +733,20 @@ def rooney_biegler_vars(data): model.asymptote.fixed = True # parmest will unfix theta variables model.rate_constant.fixed = True - model.hour = pyo.Param(within=pyo.PositiveReals, mutable=True) - model.y = pyo.Param(within=pyo.PositiveReals, mutable=True) + # Add the experiment inputs + model.h = pyo.Var(initialize=data["hour"].iloc[0], bounds=(0, 10)) - def response_rule(m, h): - expr = m.asymptote * (1 - pyo.exp(-m.rate_constant * h)) - return expr + # Fix the experiment inputs + model.h.fix() - model.response_function = pyo.Expression(data.hour, rule=response_rule) + # Add experiment outputs + model.y = pyo.Var(initialize=data['y'].iloc[0], within=pyo.PositiveReals) + + # Define the model equations + def response_rule(m): + return m.y == m.asymptote * (1 - pyo.exp(-m.rate_constant * m.h)) + + model.response_con = pyo.Constraint(rule=response_rule) return model @@ -775,14 +761,14 @@ def label_model(self): m = self.model m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) - m.experiment_outputs.update( - [(m.hour, self.data["hour"]), (m.y, self.data["y"])] - ) + m.experiment_outputs.update([(m.y, self.data["y"])]) m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) m.unknown_parameters.update( (k, pyo.ComponentUID(k)) for k in [m.asymptote, m.rate_constant] ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) rooney_biegler_vars_exp_list = [] for i in range(self.data.shape[0]): @@ -802,16 +788,22 @@ def rooney_biegler_indexed_vars(data): ) model.theta["rate_constant"].fixed = True - model.hour = pyo.Param(within=pyo.PositiveReals, mutable=True) - model.y = pyo.Param(within=pyo.PositiveReals, mutable=True) + # Add the experiment inputs + model.h = pyo.Var(initialize=data["hour"].iloc[0], bounds=(0, 10)) - def response_rule(m, h): - expr = m.theta["asymptote"] * ( - 1 - pyo.exp(-m.theta["rate_constant"] * h) + # Fix the experiment inputs + model.h.fix() + + # Add experiment outputs + model.y = pyo.Var(initialize=data['y'].iloc[0], within=pyo.PositiveReals) + + # Define the model equations + def response_rule(m): + return m.y == m.theta["asymptote"] * ( + 1 - pyo.exp(-m.theta["rate_constant"] * m.h) ) - return expr - model.response_function = pyo.Expression(data.hour, rule=response_rule) + model.response_con = pyo.Constraint(rule=response_rule) return model @@ -826,28 +818,21 @@ def label_model(self): m = self.model m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) - m.experiment_outputs.update( - [(m.hour, self.data["hour"]), (m.y, self.data["y"])] - ) + m.experiment_outputs.update([(m.y, self.data["y"])]) m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) m.unknown_parameters.update((k, pyo.ComponentUID(k)) for k in [m.theta]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + rooney_biegler_indexed_vars_exp_list = [] for i in range(self.data.shape[0]): rooney_biegler_indexed_vars_exp_list.append( RooneyBieglerExperimentIndexedVars(self.data.loc[i, :]) ) - # Sum of squared error function - def SSE(model): - expr = ( - model.experiment_outputs[model.y] - - model.response_function[model.experiment_outputs[model.hour]] - ) ** 2 - return expr - - self.objective_function = SSE + self.objective_function = 'SSE' theta_vals = pd.DataFrame([20, 1], index=["asymptote", "rate_constant"]).T theta_vals_index = pd.DataFrame( @@ -899,16 +884,16 @@ def check_rooney_biegler_results(self, objval, cov): self.assertAlmostEqual(objval, 4.3317112, places=2) self.assertAlmostEqual( - cov.iloc[asymptote_index, asymptote_index], 6.30579403, places=2 + cov.iloc[asymptote_index, asymptote_index], 6.155892, places=2 ) # 6.22864 from paper self.assertAlmostEqual( - cov.iloc[asymptote_index, rate_constant_index], -0.4395341, places=2 + cov.iloc[asymptote_index, rate_constant_index], -0.425232, places=2 ) # -0.4322 from paper self.assertAlmostEqual( - cov.iloc[rate_constant_index, asymptote_index], -0.4395341, places=2 + cov.iloc[rate_constant_index, asymptote_index], -0.425232, places=2 ) # -0.4322 from paper self.assertAlmostEqual( - cov.iloc[rate_constant_index, rate_constant_index], 0.04193591, places=2 + cov.iloc[rate_constant_index, rate_constant_index], 0.040571, places=2 ) # 0.04124 from paper @unittest.skipUnless(pynumero_ASL_available, 'pynumero_ASL is not available') @@ -918,8 +903,13 @@ def test_parmest_basics(self): pest = parmest.Estimator( parmest_input["exp_list"], obj_function=self.objective_function ) - - objval, thetavals, cov = pest.theta_est(calc_cov=True, cov_n=6) + # estimate the parameters and covariance matrix + objval, thetavals = pest.theta_est() + # For covariance, using reduced_hessian method since finite difference + # and automatic differentiation may differ from paper results in the + # 3rd decimal place, likely due to differences in finite difference + # approximation of the Jacobian + cov = pest.cov_est(method="reduced_hessian") self.check_rooney_biegler_results(objval, cov) obj_at_theta = pest.objective_at_theta(parmest_input["theta_vals"]) @@ -933,7 +923,8 @@ def test_parmest_basics_with_initialize_parmest_model_option(self): parmest_input["exp_list"], obj_function=self.objective_function ) - objval, thetavals, cov = pest.theta_est(calc_cov=True, cov_n=6) + objval, thetavals = pest.theta_est() + cov = pest.cov_est(method="reduced_hessian") self.check_rooney_biegler_results(objval, cov) obj_at_theta = pest.objective_at_theta( @@ -954,7 +945,8 @@ def test_parmest_basics_with_square_problem_solve(self): parmest_input["theta_vals"], initialize_parmest_model=True ) - objval, thetavals, cov = pest.theta_est(calc_cov=True, cov_n=6) + objval, thetavals = pest.theta_est() + cov = pest.cov_est(method="reduced_hessian") self.check_rooney_biegler_results(objval, cov) self.assertAlmostEqual(obj_at_theta["obj"][0], 16.531953, places=2) @@ -970,7 +962,8 @@ def test_parmest_basics_with_square_problem_solve_no_theta_vals(self): obj_at_theta = pest.objective_at_theta(initialize_parmest_model=True) - objval, thetavals, cov = pest.theta_est(calc_cov=True, cov_n=6) + objval, thetavals = pest.theta_est() + cov = pest.cov_est(method="reduced_hessian") self.check_rooney_biegler_results(objval, cov) @@ -1107,7 +1100,8 @@ def _dccrate(m, t): def ComputeFirstStageCost_rule(m): return 0 - m.FirstStageCost = pyo.Expression(rule=ComputeFirstStageCost_rule) + # Model objective component names adjusted to prevent reserved name error. + m.FirstStage = pyo.Expression(rule=ComputeFirstStageCost_rule) def ComputeSecondStageCost_rule(m): return sum( @@ -1117,14 +1111,12 @@ def ComputeSecondStageCost_rule(m): for t in meas_t ) - m.SecondStageCost = pyo.Expression(rule=ComputeSecondStageCost_rule) + m.SecondStage = pyo.Expression(rule=ComputeSecondStageCost_rule) def total_cost_rule(model): - return model.FirstStageCost + model.SecondStageCost + return model.FirstStage + model.SecondStage - m.Total_Cost_Objective = pyo.Objective( - rule=total_cost_rule, sense=pyo.minimize - ) + m.Total_Cost = pyo.Objective(rule=total_cost_rule, sense=pyo.minimize) disc = pyo.TransformationFactory("dae.collocation") disc.apply_to(m, nfe=20, ncp=2) @@ -1165,6 +1157,10 @@ def label_model(self): m.unknown_parameters.update( (k, pyo.ComponentUID(k)) for k in [m.k1, m.k2] ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update((m.ca[t], None) for t in meas_time_points) + m.measurement_error.update((m.cb[t], None) for t in meas_time_points) + m.measurement_error.update((m.cc[t], None) for t in meas_time_points) def get_labeled_model(self): self.create_model() @@ -1210,8 +1206,8 @@ def get_labeled_model(self): exp_list_df = [ReactorDesignExperimentDAE(data_df)] exp_list_dict = [ReactorDesignExperimentDAE(data_dict)] - self.pest_df = parmest.Estimator(exp_list_df) - self.pest_dict = parmest.Estimator(exp_list_dict) + self.pest_df = parmest.Estimator(exp_list_df, obj_function="SSE") + self.pest_dict = parmest.Estimator(exp_list_dict, obj_function="SSE") # Estimator object with multiple scenarios exp_list_df_multiple = [ @@ -1223,8 +1219,12 @@ def get_labeled_model(self): ReactorDesignExperimentDAE(data_dict), ] - self.pest_df_multiple = parmest.Estimator(exp_list_df_multiple) - self.pest_dict_multiple = parmest.Estimator(exp_list_dict_multiple) + self.pest_df_multiple = parmest.Estimator( + exp_list_df_multiple, obj_function="SSE" + ) + self.pest_dict_multiple = parmest.Estimator( + exp_list_dict_multiple, obj_function="SSE" + ) # Create an instance of the model self.m_df = ABC_model(data_df) @@ -1315,10 +1315,11 @@ def test_covariance(self): # 3 data components (ca, cb, cc), 20 timesteps, 1 scenario = 60 # In this example, this is the number of data points in data_df, but that's # only because the data is indexed by time and contains no additional information. - n = 60 + n = 20 # Compute covariance using parmest - obj, theta, cov = self.pest_df.theta_est(calc_cov=True, cov_n=n) + obj, theta = self.pest_df.theta_est() + cov = self.pest_df.cov_est(method="reduced_hessian") # Compute covariance using interior_point vars_list = [self.m_df.k1, self.m_df.k2] @@ -1416,6 +1417,509 @@ def test_theta_est_with_square_initialization_diagnostic_mode_true(self): self.pest.diagnostic_mode = False +class LinearThetaExperiment(Experiment): + def __init__(self, x, y, include_second_output=False): + self.x_data = x + self.y_data = y + self.include_second_output = include_second_output + self.model = None + + def create_model(self): + m = pyo.ConcreteModel() + m.theta = pyo.Var(initialize=0.0, bounds=(-10.0, 10.0)) + m.x = pyo.Param(initialize=float(self.x_data), mutable=False) + m.y = pyo.Var(initialize=float(self.y_data)) + m.y_link = pyo.Constraint(expr=m.y == m.theta + m.x) + if self.include_second_output: + m.z = pyo.Var(initialize=2.0 * self.y_data) + m.z_link = pyo.Constraint(expr=m.z == 2.0 * m.theta + m.x) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, float(self.y_data))]) + if self.include_second_output: + m.experiment_outputs.update([(m.z, float(2.0 * self.y_data))]) + + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update([(m.theta, pyo.ComponentUID(m.theta))]) + + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + if self.include_second_output: + m.measurement_error.update([(m.z, None)]) + + def get_labeled_model(self): + self.create_model() + self.label_model() + return self.model + + +def _build_estimator(data, include_second_output=False): + exp_list = [ + LinearThetaExperiment(x=x, y=y, include_second_output=include_second_output) + for x, y in data + ] + return parmest.Estimator(exp_list, obj_function="SSE") + + +@unittest.skipIf( + not parmest.parmest_available, + "Cannot test parmest: required dependencies are missing", +) +class TestParmestBlockEF(unittest.TestCase): + def test_block_ef_structure_counts(self): + pest = _build_estimator([(1.0, 2.0), (2.0, 4.0)]) + model = pest._create_scenario_blocks() + + theta_names = model._parmest_theta_names + self.assertEqual(len(list(model.exp_scenarios.keys())), 2) + self.assertEqual(len(model.theta_link_constraints), 2 * len(theta_names)) + self.assertTrue(hasattr(model, "Obj")) + for block in model.exp_scenarios.values(): + self.assertFalse(block.Total_Cost_Objective.active) + + def test_block_isolation_no_component_leakage(self): + pest = _build_estimator([(1.0, 2.0), (5.0, 6.0)]) + model = pest._create_scenario_blocks() + + block0 = model.exp_scenarios[0] + block1 = model.exp_scenarios[1] + self.assertIsNot(block0.y, block1.y) + block0.y.set_value(123.0) + self.assertNotEqual(pyo.value(block1.y), 123.0) + self.assertNotEqual(pyo.value(block0.x), pyo.value(block1.x)) + + def test_fix_theta_sets_all_scenario_theta_values(self): + pest = _build_estimator([(1.0, 2.0), (2.0, 4.0)]) + model = pest._create_scenario_blocks(theta_vals={"theta": 1.0}, fix_theta=True) + + self.assertTrue(model.parmest_theta["theta"].fixed) + self.assertAlmostEqual(pyo.value(model.parmest_theta["theta"]), 1.0, places=10) + for block in model.exp_scenarios.values(): + self.assertTrue(block.theta.fixed) + self.assertAlmostEqual(pyo.value(block.theta), 1.0, places=10) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_objective_at_theta_fixed_value(self): + pest = _build_estimator([(1.0, 2.0), (2.0, 4.0)]) + theta_values = pd.DataFrame([[1.0]], columns=["theta"]) + obj_at_theta = pest.objective_at_theta(theta_values=theta_values) + # residuals at theta=1 are [0, 1], objective is averaged over two scenarios + self.assertAlmostEqual(obj_at_theta.loc[0, "obj"], 0.5, places=8) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_objective_at_theta_none_uses_initial_theta(self): + pest = _build_estimator([(1.0, 2.0), (2.0, 3.0)]) + obj_at_theta = pest.objective_at_theta() + # with theta initialized to 0, predictions are [1,2], residuals [1,1], avg objective 1 + self.assertAlmostEqual(obj_at_theta.loc[0, "obj"], 1.0, places=8) + self.assertAlmostEqual(obj_at_theta.loc[0, "theta"], 0.0, places=8) + + def test_invalid_solver_name_raises_runtimeerror(self): + pest = _build_estimator([(1.0, 2.0), (2.0, 4.0)]) + with self.assertRaisesRegex( + RuntimeError, "Unknown solver in Q_Opt=not_a_solver" + ): + pest.theta_est(solver="not_a_solver") + + def test_theta_values_duplicate_columns_rejected(self): + pest = _build_estimator([(1.0, 2.0), (2.0, 4.0)]) + duplicate_cols = pd.DataFrame([[1.0, 2.0]], columns=["theta", "theta"]) + with self.assertRaisesRegex( + ValueError, "Duplicate theta names are not allowed" + ): + pest.objective_at_theta(theta_values=duplicate_cols) + + def test_count_total_experiments_multi_output(self): + exp_list = [ + LinearThetaExperiment(1.0, 2.0, include_second_output=True), + LinearThetaExperiment(2.0, 4.0, include_second_output=True), + ] + total_points = parmest._count_total_experiments(exp_list) + # The current parmest convention counts datapoints for one output family. + self.assertEqual(total_points, 2) + + +@unittest.skipIf( + not parmest.parmest_available, + "Cannot test parmest regularization: required dependencies are missing", +) +class TestRegularizationCore(unittest.TestCase): + # These tests intentionally use a tiny linear model so each expected + # regularization term can be computed analytically and reviewed quickly. + class LinearExperiment(Experiment): + def __init__(self, x, y, theta0_init=0.0, theta1_init=0.0): + self.x = float(x) + self.y = float(y) + self.theta0_init = float(theta0_init) + self.theta1_init = float(theta1_init) + super().__init__(model=None) + self.create_model() + self.label_model() + + def create_model(self): + m = pyo.ConcreteModel() + m.theta0 = pyo.Var(initialize=self.theta0_init) + m.theta1 = pyo.Var(initialize=self.theta1_init) + m.x = pyo.Param(initialize=self.x, mutable=True) + m.pred = pyo.Var(initialize=self.y) + m.pred_link = pyo.Constraint(expr=m.pred == m.theta0 + m.theta1 * m.x) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.pred, self.y)]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + (k, pyo.ComponentUID(k)) for k in [m.theta0, m.theta1] + ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.pred, None)]) + + class DummyExperiment(Experiment): + def __init__(self): + m = pyo.ConcreteModel() + m.theta0 = pyo.Var(initialize=0.0) + m.theta1 = pyo.Var(initialize=0.0) + m.pred = pyo.Var(initialize=0.0) + m.pred_link = pyo.Constraint(expr=m.pred == m.theta0 + m.theta1) + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.pred, 0.0)]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + (k, pyo.ComponentUID(k)) for k in [m.theta0, m.theta1] + ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.pred, None)]) + super().__init__(model=m) + + @staticmethod + def _make_var_labeled_model(y_obs=5.0): + # Var-based helper used for direct objective-expression checks. + m = pyo.ConcreteModel() + m.theta0 = pyo.Var(initialize=0.0) + m.theta1 = pyo.Var(initialize=0.0) + m.pred = pyo.Expression(expr=m.theta0 + 2.0 * m.theta1) + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.pred, float(y_obs))]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + (k, pyo.ComponentUID(k)) for k in [m.theta0, m.theta1] + ) + return m + + @staticmethod + def _obj_at_theta(pest, theta0, theta1): + # Evaluate objective for a single theta row to keep assertions explicit. + theta = pd.DataFrame([[theta0, theta1]], columns=["theta0", "theta1"]) + return pest.objective_at_theta(theta_values=theta).iloc[0]["obj"] + + def test_l2_objective_value_matches_manual_quadratic(self): + m = self._make_var_labeled_model(y_obs=5.0) + m.theta0.set_value(4.0) + m.theta1.set_value(-1.0) + + prior_fim = pd.DataFrame( + [[2.0, 0.0], [0.0, 4.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + theta_ref = pd.Series({"theta0": 1.0, "theta1": 2.0}) + weight = 3.0 + + expr = parmest.L2_regularized_objective( + m, + prior_FIM=prior_fim, + theta_ref=theta_ref, + regularization_weight=weight, + obj_function=parmest.SSE, + ) + + sse_expected = 9.0 + l2_expected = 27.0 + expected = sse_expected + weight * l2_expected + + self.assertAlmostEqual(pyo.value(expr), expected) + + def test_l2_penalty_not_double_counted_across_scenarios(self): + # Confirms regularization is applied once at the estimator level, + # not once per scenario. + exp_list = [self.LinearExperiment(1.0, 1.0), self.LinearExperiment(2.0, 2.0)] + prior_fim = pd.DataFrame( + [[0.0, 0.0], [0.0, 2.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + theta_ref = pd.Series({"theta0": 0.0, "theta1": 0.0}) + + pest = parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L2", + prior_FIM=prior_fim, + theta_ref=theta_ref, + regularization_weight=1.0, + ) + + obj_val = self._obj_at_theta(pest, 0.0, 1.0) + self.assertAlmostEqual(obj_val, 1.0) + + def test_regularization_requires_explicit_option_when_prior_supplied(self): + # Guardrail: passing prior/FIM arguments without selecting a + # regularization mode should fail fast. + exp_list = [self.LinearExperiment(1.0, 1.0)] + prior_fim = pd.DataFrame( + [[1.0, 0.0], [0.0, 1.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + + with pytest.raises( + ValueError, match="regularization must be set when supplying prior_FIM" + ): + parmest.Estimator(exp_list, obj_function="SSE", prior_FIM=prior_fim) + + def test_l2_regularization_requires_prior_fim(self): + exp_list = [self.LinearExperiment(1.0, 1.0)] + + with pytest.raises(ValueError, match="prior_FIM must be provided"): + parmest.Estimator(exp_list, obj_function="SSE", regularization="L2") + + def test_user_specified_unsupported_regularization_raises(self): + exp_list = [self.LinearExperiment(1.0, 1.0)] + + with pytest.raises(TypeError, match="regularization must be None or one of"): + parmest.Estimator( + exp_list, obj_function="SSE", regularization=lambda m: m.theta0**2 + ) + + def test_l2_lambda_zero_matches_unregularized_objective(self): + exp_list = [self.LinearExperiment(1.0, 1.0), self.LinearExperiment(2.0, 2.0)] + prior_fim = pd.DataFrame( + [[2.0, 0.0], [0.0, 1.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + theta_ref = pd.Series({"theta0": 0.0, "theta1": 0.0}) + + pest_base = parmest.Estimator(exp_list, obj_function="SSE") + pest_l2_zero = parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L2", + prior_FIM=prior_fim, + theta_ref=theta_ref, + regularization_weight=0.0, + ) + + for theta0, theta1 in [(0.0, 0.0), (0.5, 1.5), (-1.0, 2.0)]: + obj_base = self._obj_at_theta(pest_base, theta0, theta1) + obj_l2_zero = self._obj_at_theta(pest_l2_zero, theta0, theta1) + self.assertAlmostEqual(obj_l2_zero, obj_base) + + def test_prior_subset_penalizes_only_selected_parameter(self): + # Prior indexed only by theta1 should leave theta0 unpenalized. + exp_list = [self.LinearExperiment(1.0, 1.0)] + prior_fim = pd.DataFrame([[4.0]], index=["theta1"], columns=["theta1"]) + + pest_base = parmest.Estimator(exp_list, obj_function="SSE") + pest_l2 = parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L2", + prior_FIM=prior_fim, + theta_ref=pd.Series({"theta1": 0.0}), + regularization_weight=1.0, + ) + + obj_base_theta0 = self._obj_at_theta(pest_base, theta0=1.0, theta1=0.0) + obj_l2_theta0 = self._obj_at_theta(pest_l2, theta0=1.0, theta1=0.0) + self.assertAlmostEqual(obj_l2_theta0, obj_base_theta0) + + obj_base_theta1 = self._obj_at_theta(pest_base, theta0=0.0, theta1=1.0) + obj_l2_theta1 = self._obj_at_theta(pest_l2, theta0=0.0, theta1=1.0) + expected_penalty = 0.5 * 4.0 * (1.0**2) + self.assertAlmostEqual(obj_l2_theta1 - obj_base_theta1, expected_penalty) + + def test_negative_regularization_weight_raises(self): + exp_list = [self.LinearExperiment(1.0, 1.0)] + prior_fim = pd.DataFrame( + [[1.0, 0.0], [0.0, 1.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + + with pytest.raises( + ValueError, match="regularization_weight must be nonnegative" + ): + parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L2", + prior_FIM=prior_fim, + regularization_weight=-1.0, + ) + + def test_missing_theta_ref_entries_raise_clear_error(self): + exp_list = [self.LinearExperiment(1.0, 1.0)] + prior_fim = pd.DataFrame( + [[1.0, 0.0], [0.0, 1.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + + pest = parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L2", + prior_FIM=prior_fim, + theta_ref=pd.Series({"theta0": 0.0}), + regularization_weight=1.0, + ) + + with pytest.raises( + ValueError, match=r"theta_ref is missing values for parameter\(s\): theta1" + ): + _ = self._obj_at_theta(pest, theta0=0.0, theta1=0.0) + + def test_non_psd_prior_fim_rejected(self): + exp_list = [self.LinearExperiment(1.0, 1.0)] + non_psd = pd.DataFrame( + [[1.0, 2.0], [2.0, -1.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + + with pytest.raises(ValueError, match="positive semi-definite"): + parmest.Estimator( + exp_list, obj_function="SSE", regularization="L2", prior_FIM=non_psd + ) + + def test_compute_covariance_matrix_adds_prior_fim_weighted(self): + exp_list = [self.DummyExperiment(), self.DummyExperiment()] + + def fake_finite_difference_FIM(*args, **kwargs): + return np.eye(2) + + prior_fim = pd.DataFrame( + [[1.0, 0.0], [0.0, 3.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + theta_vals = {"theta0": 0.0, "theta1": 0.0} + + # Replace finite-difference FIM with identity so this test isolates + # only the prior_FIM + regularization_weight addition path. + original_finite_difference_FIM = parmest._finite_difference_FIM + try: + parmest._finite_difference_FIM = fake_finite_difference_FIM + cov = parmest.compute_covariance_matrix( + experiment_list=exp_list, + method=parmest.CovarianceMethod.finite_difference.value, + obj_function=parmest.SSE, + theta_vals=theta_vals, + step=1e-3, + solver="ipopt", + tee=False, + prior_FIM=prior_fim, + regularization_weight=2.0, + ) + finally: + # Always restore global function to avoid cross-test contamination. + parmest._finite_difference_FIM = original_finite_difference_FIM + + self.assertAlmostEqual(cov.loc["theta0", "theta0"], 0.25) + self.assertAlmostEqual(cov.loc["theta1", "theta1"], 0.125) + self.assertAlmostEqual(cov.loc["theta0", "theta1"], 0.0) + self.assertAlmostEqual(cov.loc["theta1", "theta0"], 0.0) + + def test_l1_smooth_objective_value_matches_manual_expression(self): + # Verifies the smooth-L1 form: sum_i w_i * sqrt(delta_i^2 + epsilon). + m = self._make_var_labeled_model(y_obs=5.0) + m.theta0.set_value(4.0) + m.theta1.set_value(-1.0) + + prior_fim = pd.DataFrame( + [[2.0, 0.0], [0.0, 4.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + theta_ref = pd.Series({"theta0": 1.0, "theta1": 2.0}) + weight = 3.0 + eps = 1e-6 + + expr = parmest.L1_regularized_objective( + m, + prior_FIM=prior_fim, + theta_ref=theta_ref, + regularization_weight=weight, + regularization_epsilon=eps, + obj_function=parmest.SSE, + ) + + sse_expected = 9.0 + l1_expected = 2.0 * np.sqrt(9.0 + eps) + 4.0 * np.sqrt(9.0 + eps) + expected = sse_expected + weight * l1_expected + + self.assertAlmostEqual(pyo.value(expr), expected) + + def test_l1_lambda_zero_matches_unregularized_objective(self): + exp_list = [self.LinearExperiment(1.0, 1.0), self.LinearExperiment(2.0, 2.0)] + prior_fim = pd.DataFrame( + [[2.0, 0.0], [0.0, 1.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + theta_ref = pd.Series({"theta0": 0.0, "theta1": 0.0}) + + pest_base = parmest.Estimator(exp_list, obj_function="SSE") + pest_l1_zero = parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L1", + prior_FIM=prior_fim, + theta_ref=theta_ref, + regularization_weight=0.0, + regularization_epsilon=1e-6, + ) + + for theta0, theta1 in [(0.0, 0.0), (0.5, 1.5), (-1.0, 2.0)]: + obj_base = self._obj_at_theta(pest_base, theta0, theta1) + obj_l1_zero = self._obj_at_theta(pest_l1_zero, theta0, theta1) + self.assertAlmostEqual(obj_l1_zero, obj_base) + + def test_l1_nonpositive_epsilon_raises(self): + exp_list = [self.LinearExperiment(1.0, 1.0)] + prior_fim = pd.DataFrame( + [[1.0, 0.0], [0.0, 1.0]], + index=["theta0", "theta1"], + columns=["theta0", "theta1"], + ) + + with pytest.raises(ValueError, match="regularization_epsilon must be positive"): + parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L1", + prior_FIM=prior_fim, + regularization_weight=1.0, + regularization_epsilon=0.0, + ) + + with pytest.raises(ValueError, match="regularization_epsilon must be positive"): + parmest.Estimator( + exp_list, + obj_function="SSE", + regularization="L1", + prior_FIM=prior_fim, + regularization_weight=1.0, + regularization_epsilon=-1e-6, + ) + + ########################### # tests for deprecated UI # ########################### diff --git a/pyomo/contrib/parmest/tests/test_parmest_multistart.py b/pyomo/contrib/parmest/tests/test_parmest_multistart.py new file mode 100644 index 00000000000..e8770e07221 --- /dev/null +++ b/pyomo/contrib/parmest/tests/test_parmest_multistart.py @@ -0,0 +1,433 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of +# Sandia, LLC Under the terms of Contract DE-NA0003525 with National +# Technology and Engineering Solutions of Sandia, LLC, the U.S. Government +# retains certain rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import math + +import pyomo.common.unittest as unittest +import pyomo.environ as pyo +from pyomo.common.dependencies import numpy as np, pandas as pd +from unittest.mock import patch + +import pyomo.contrib.parmest.parmest as parmest +from pyomo.contrib.parmest.experiment import Experiment + +ipopt_available = pyo.SolverFactory("ipopt").available() + + +class LinearThetaExperiment(Experiment): + def __init__(self, x, y): + self.x_data = x + self.y_data = y + self.model = None + + def create_model(self): + m = pyo.ConcreteModel() + m.theta = pyo.Var(initialize=0.0, bounds=(-10.0, 10.0)) + m.x = pyo.Param(initialize=float(self.x_data), mutable=False) + m.y = pyo.Var(initialize=float(self.y_data)) + m.eq = pyo.Constraint(expr=m.y == m.theta + m.x) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, float(self.y_data))]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update([(m.theta, pyo.ComponentUID(m.theta))]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + + def get_labeled_model(self): + self.create_model() + self.label_model() + return self.model + + +class IndexedThetaExperiment(Experiment): + def __init__(self): + self.model = None + + def create_model(self): + m = pyo.ConcreteModel() + m.I = pyo.Set(initialize=["a", "b"]) + m.theta = pyo.Var(m.I, initialize={"a": 1.0, "b": 2.0}) + m.theta["a"].setlb(0.0) + m.theta["a"].setub(5.0) + m.theta["b"].setlb(0.0) + m.theta["b"].setub(5.0) + m.theta["a"].fix() + m.theta["b"].fix() + m.y = pyo.Var(initialize=3.0) + m.eq = pyo.Constraint(expr=m.y == m.theta["a"] + m.theta["b"]) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, 3.0)]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update([(m.theta, pyo.ComponentUID(m.theta))]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + + def get_labeled_model(self): + self.create_model() + self.label_model() + return self.model + + +class NoBoundsExperiment(Experiment): + def __init__(self): + self.model = None + + def create_model(self): + m = pyo.ConcreteModel() + m.theta = pyo.Var(initialize=1.0) + m.y = pyo.Var(initialize=2.0) + m.eq = pyo.Constraint(expr=m.y == m.theta + 1.0) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, 2.0)]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update([(m.theta, pyo.ComponentUID(m.theta))]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + + def get_labeled_model(self): + self.create_model() + self.label_model() + return self.model + + +class StartCoupledExperiment(Experiment): + """ + Model intentionally couples a fixed term ("bias") to theta_initial at + build time. This exposes stale-model bugs in multistart paths. + """ + + def __init__(self, theta_initial=None): + self.theta_initial = ( + theta_initial if theta_initial is not None else {"theta": 0.0} + ) + self.model = None + + def create_model(self): + m = pyo.ConcreteModel() + m.theta = pyo.Var( + initialize=float(self.theta_initial["theta"]), bounds=(-10.0, 10.0) + ) + m.bias = pyo.Param(initialize=float(self.theta_initial["theta"]), mutable=False) + m.y = pyo.Var(initialize=0.0) + m.eq = pyo.Constraint(expr=m.y == m.theta + m.bias) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, 0.0)]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update([(m.theta, pyo.ComponentUID(m.theta))]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + + def get_labeled_model(self): + if self.model is None: + self.create_model() + self.label_model() + return self.model + + +def _build_linear_estimator(): + exp_list = [LinearThetaExperiment(1.0, 2.0), LinearThetaExperiment(2.0, 3.0)] + return parmest.Estimator(exp_list, obj_function="SSE") + + +@unittest.skipIf( + not parmest.parmest_available, + "Cannot test parmest: required dependencies are missing", +) +class TestParmestMultistart(unittest.TestCase): + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_multistart_baseline_equivalence_n1(self): + pest = _build_linear_estimator() + obj1, theta1 = pest.theta_est() + _, best_theta, best_obj = pest.theta_est_multistart( + n_restarts=1, multistart_sampling_method="uniform_random", seed=7 + ) + self.assertAlmostEqual(obj1, best_obj, places=7) + self.assertAlmostEqual(theta1["theta"], best_theta["theta"], places=7) + + def test_uniform_sampling_is_deterministic_with_seed(self): + pest = _build_linear_estimator() + df1 = pest._generate_initial_theta( + seed=4, n_restarts=5, multistart_sampling_method="uniform_random" + ) + df2 = pest._generate_initial_theta( + seed=4, n_restarts=5, multistart_sampling_method="uniform_random" + ) + self.assertTrue(df1[["theta"]].equals(df2[["theta"]])) + + def test_uniform_sampling_changes_with_different_seed(self): + pest = _build_linear_estimator() + df1 = pest._generate_initial_theta( + seed=4, n_restarts=5, multistart_sampling_method="uniform_random" + ) + df2 = pest._generate_initial_theta( + seed=5, n_restarts=5, multistart_sampling_method="uniform_random" + ) + self.assertFalse(df1[["theta"]].equals(df2[["theta"]])) + + def test_latin_hypercube_sampling_is_deterministic(self): + pest = _build_linear_estimator() + df1 = pest._generate_initial_theta( + seed=11, n_restarts=4, multistart_sampling_method="latin_hypercube" + ) + df2 = pest._generate_initial_theta( + seed=11, n_restarts=4, multistart_sampling_method="latin_hypercube" + ) + self.assertTrue(df1[["theta"]].equals(df2[["theta"]])) + + def test_sobol_sampling_is_deterministic(self): + pest = _build_linear_estimator() + df1 = pest._generate_initial_theta( + seed=12, n_restarts=4, multistart_sampling_method="sobol_sampling" + ) + df2 = pest._generate_initial_theta( + seed=12, n_restarts=4, multistart_sampling_method="sobol_sampling" + ) + self.assertTrue(df1[["theta"]].equals(df2[["theta"]])) + + def test_generated_starts_are_within_bounds(self): + pest = _build_linear_estimator() + for method in ("uniform_random", "latin_hypercube", "sobol_sampling"): + df = pest._generate_initial_theta( + seed=1, n_restarts=8, multistart_sampling_method=method + ) + self.assertTrue(((df["theta"] >= -10.0) & (df["theta"] <= 10.0)).all()) + + def test_missing_bounds_raise_error(self): + pest = parmest.Estimator([NoBoundsExperiment()], obj_function="SSE") + with self.assertRaisesRegex( + ValueError, "lower and upper bounds for the theta values must be defined" + ): + pest._generate_initial_theta( + seed=1, n_restarts=2, multistart_sampling_method="uniform_random" + ) + + def test_invalid_bounds_raise_error(self): + class InvalidBoundsExperiment(Experiment): + def __init__(self): + self.model = None + + def create_model(self): + m = pyo.ConcreteModel() + m.theta = pyo.Var(initialize=1.0) + m.theta.setlb(2.0) + m.theta.setub(1.0) + m.y = pyo.Var(initialize=2.0) + m.eq = pyo.Constraint(expr=m.y == m.theta + 1.0) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, 2.0)]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update([(m.theta, pyo.ComponentUID(m.theta))]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + + def get_labeled_model(self): + self.create_model() + self.label_model() + return self.model + + pest = parmest.Estimator([InvalidBoundsExperiment()], obj_function="SSE") + with self.assertRaisesRegex(ValueError, "lower bound must be less than"): + pest._generate_initial_theta( + seed=1, n_restarts=2, multistart_sampling_method="uniform_random" + ) + + def test_user_provided_values_dimension_mismatch_raises(self): + pest = _build_linear_estimator() + user_df = pd.DataFrame([[1.0, 2.0]], columns=["theta", "extra"]) + with self.assertRaisesRegex(ValueError, "exactly one column per theta name"): + pest.theta_est_multistart( + n_restarts=1, + multistart_sampling_method="user_provided_values", + user_provided_df=user_df, + ) + + def test_user_provided_values_column_order_maps_by_name(self): + pest = parmest.Estimator([IndexedThetaExperiment()], obj_function="SSE") + user_df = pd.DataFrame( + [[0.3, 4.2], [0.4, 4.1]], columns=["theta[b]", "theta[a]"] + ) + results_df, _, _ = pest.theta_est_multistart( + n_restarts=2, + multistart_sampling_method="user_provided_values", + user_provided_df=user_df, + ) + self.assertAlmostEqual(results_df.loc[0, "theta[a]"], 4.2, places=12) + self.assertAlmostEqual(results_df.loc[0, "theta[b]"], 0.3, places=12) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_state_isolation_between_starts(self): + pest = _build_linear_estimator() + init = pd.DataFrame([[-9.0], [9.0]], columns=["theta"]) + results_df, _, _ = pest.theta_est_multistart( + user_provided_df=init, save_results=False + ) + # Initial starts should remain exactly as supplied. + self.assertAlmostEqual(results_df.loc[0, "theta"], -9.0, places=12) + self.assertAlmostEqual(results_df.loc[1, "theta"], 9.0, places=12) + # Both runs converge to the same optimum, showing no cross-start contamination. + self.assertAlmostEqual( + results_df.loc[0, "converged_theta"], + results_df.loc[1, "converged_theta"], + places=8, + ) + + def test_one_start_failure_returns_best_feasible(self): + pest = _build_linear_estimator() + theta_values = pd.DataFrame([[-1.0], [2.0]], columns=["theta"]) + + def fake_q_opt(*args, **kwargs): + theta = kwargs["theta_vals"]["theta"] + if theta < 0: + raise RuntimeError("boom") + return 1.25, {"theta": 1.0}, pyo.TerminationCondition.optimal + + with patch.object(pest, "_Q_opt", side_effect=fake_q_opt): + results_df, best_theta, best_obj = pest.theta_est_multistart( + user_provided_df=theta_values, save_results=False + ) + + self.assertTrue( + str(results_df.loc[0, "solver termination"]).startswith("exception(start=0") + ) + self.assertAlmostEqual(best_obj, 1.25, places=12) + self.assertAlmostEqual(best_theta["theta"], 1.0, places=12) + + def test_all_starts_fail_returns_diagnostics(self): + pest = _build_linear_estimator() + theta_values = pd.DataFrame([[1.0], [2.0]], columns=["theta"]) + + def fake_q_opt(*args, **kwargs): + raise RuntimeError("all failed") + + with patch.object(pest, "_Q_opt", side_effect=fake_q_opt): + results_df, best_theta, best_obj = pest.theta_est_multistart( + user_provided_df=theta_values, save_results=False + ) + + self.assertIsNone(best_theta) + self.assertTrue(math.isnan(best_obj)) + self.assertTrue( + results_df["solver termination"] + .astype(str) + .str.contains("exception\\(start=", regex=True) + .all() + ) + + def test_best_selection_filters_nonoptimal_status(self): + pest = _build_linear_estimator() + theta_values = pd.DataFrame([[1.0], [2.0]], columns=["theta"]) + + def fake_q_opt(*args, **kwargs): + theta = kwargs["theta_vals"]["theta"] + if theta < 1.5: + return 0.1, {"theta": 0.1}, pyo.TerminationCondition.maxIterations + return 0.2, {"theta": 0.2}, pyo.TerminationCondition.optimal + + with patch.object(pest, "_Q_opt", side_effect=fake_q_opt): + _, best_theta, best_obj = pest.theta_est_multistart( + user_provided_df=theta_values, save_results=False + ) + + self.assertAlmostEqual(best_obj, 0.2, places=12) + self.assertAlmostEqual(best_theta["theta"], 0.2, places=12) + + def test_tie_breaking_is_deterministic_first_index(self): + pest = _build_linear_estimator() + theta_values = pd.DataFrame([[5.0], [6.0], [7.0]], columns=["theta"]) + + def fake_q_opt(*args, **kwargs): + theta = kwargs["theta_vals"]["theta"] + return 1.0, {"theta": theta}, pyo.TerminationCondition.optimal + + with patch.object(pest, "_Q_opt", side_effect=fake_q_opt): + _, best_theta, best_obj = pest.theta_est_multistart( + user_provided_df=theta_values, save_results=False + ) + + self.assertAlmostEqual(best_obj, 1.0, places=12) + self.assertAlmostEqual(best_theta["theta"], 5.0, places=12) + + def test_indexed_unknown_parameters_supported_in_sampling(self): + pest = parmest.Estimator([IndexedThetaExperiment()], obj_function="SSE") + df = pest._generate_initial_theta( + seed=10, n_restarts=3, multistart_sampling_method="uniform_random" + ) + self.assertTrue({"theta[a]", "theta[b]"}.issubset(set(df.columns))) + + def test_count_total_experiments_uses_one_output_family(self): + class MultiOutputExperiment(Experiment): + def create_model(self): + m = pyo.ConcreteModel() + m.theta = pyo.Var(initialize=0.0, bounds=(-10, 10)) + m.y = pyo.Var(initialize=1.0) + m.z = pyo.Var(initialize=2.0) + m.c1 = pyo.Constraint(expr=m.y == m.theta + 1.0) + m.c2 = pyo.Constraint(expr=m.z == 2.0 * m.theta + 2.0) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, 1.0), (m.z, 2.0)]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update([(m.theta, pyo.ComponentUID(m.theta))]) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None), (m.z, None)]) + + def get_labeled_model(self): + self.create_model() + self.label_model() + return self.model + + total_points = parmest._count_total_experiments( + [MultiOutputExperiment(), MultiOutputExperiment()] + ) + self.assertEqual(total_points, 2) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_multistart_results_reproducible_when_rerun_from_recorded_init(self): + pest = parmest.Estimator([StartCoupledExperiment()], obj_function="SSE") + init_df = pd.DataFrame([[-2.0], [1.5], [3.0]], columns=["theta"]) + results_df, _, _ = pest.theta_est_multistart( + user_provided_df=init_df, save_results=False + ) + + for _, row in results_df.iterrows(): + theta_init = {"theta": float(row["theta"])} + exp = StartCoupledExperiment(theta_initial=theta_init) + rerun = parmest.Estimator([exp], obj_function="SSE") + obj, theta = rerun.theta_est() + + self.assertTrue( + np.isclose(obj, row["final objective"], rtol=1e-6, atol=1e-8) + ) + self.assertTrue( + np.isclose(theta["theta"], row["converged_theta"], rtol=1e-6, atol=1e-8) + ) diff --git a/pyomo/contrib/parmest/tests/test_parmest_profile.py b/pyomo/contrib/parmest/tests/test_parmest_profile.py new file mode 100644 index 00000000000..6e63af13e73 --- /dev/null +++ b/pyomo/contrib/parmest/tests/test_parmest_profile.py @@ -0,0 +1,273 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of +# Sandia, LLC Under the terms of Contract DE-NA0003525 with National +# Technology and Engineering Solutions of Sandia, LLC, the U.S. Government +# retains certain rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import pyomo.common.unittest as unittest +import pyomo.environ as pyo +from pyomo.common.dependencies import numpy as np, pandas as pd +from unittest.mock import patch + +import pyomo.contrib.parmest.parmest as parmest +from pyomo.contrib.parmest.experiment import Experiment + +ipopt_available = pyo.SolverFactory("ipopt").available() + + +class AffineTwoThetaExperiment(Experiment): + def __init__(self, x, y): + self.x_data = x + self.y_data = y + self.model = None + + def create_model(self): + m = pyo.ConcreteModel() + m.theta_a = pyo.Var(initialize=1.0, bounds=(0.0, 4.0)) + m.theta_b = pyo.Var(initialize=0.0, bounds=(-3.0, 3.0)) + m.x = pyo.Param(initialize=float(self.x_data), mutable=False) + m.y = pyo.Var(initialize=float(self.y_data)) + m.eq = pyo.Constraint(expr=m.y == m.theta_a * m.x + m.theta_b) + self.model = m + + def label_model(self): + m = self.model + m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.experiment_outputs.update([(m.y, float(self.y_data))]) + m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.unknown_parameters.update( + [ + (m.theta_a, pyo.ComponentUID(m.theta_a)), + (m.theta_b, pyo.ComponentUID(m.theta_b)), + ] + ) + m.measurement_error = pyo.Suffix(direction=pyo.Suffix.LOCAL) + m.measurement_error.update([(m.y, None)]) + + def get_labeled_model(self): + self.create_model() + self.label_model() + return self.model + + +def _build_two_theta_estimator(): + exp_list = [ + AffineTwoThetaExperiment(1.0, 3.0), + AffineTwoThetaExperiment(2.0, 5.0), + AffineTwoThetaExperiment(3.0, 7.0), + ] + return parmest.Estimator(exp_list, obj_function="SSE") + + +@unittest.skipIf( + not parmest.parmest_available, + "Cannot test parmest: required dependencies are missing", +) +class TestParmestProfileLikelihood(unittest.TestCase): + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_profile_contains_baseline_minimum_point(self): + pest = _build_two_theta_estimator() + res = pest.profile_likelihood( + "theta_a", grid=[1.5, 2.0, 2.5], solver="ef_ipopt" + ) + prof = res["profiles"] + self.assertAlmostEqual(res["baseline"]["obj_hat"], prof["obj"].min(), places=8) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_profile_partial_fix_enforced(self): + pest = _build_two_theta_estimator() + res = pest.profile_likelihood( + "theta_a", grid=[1.5, 2.0, 2.5], solver="ef_ipopt" + ) + prof = res["profiles"] + self.assertTrue(np.allclose(prof["theta_value"], prof["theta__theta_a"])) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_profile_other_thetas_unfixed(self): + pest = _build_two_theta_estimator() + res = pest.profile_likelihood( + "theta_a", grid=[1.5, 2.0, 2.5], solver="ef_ipopt" + ) + prof = res["profiles"].sort_values("theta_value") + self.assertGreater( + prof["theta__theta_b"].max() - prof["theta__theta_b"].min(), 1e-6 + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_profile_repeatability_same_seed(self): + pest = _build_two_theta_estimator() + res1 = pest.profile_likelihood( + "theta_a", grid=[1.5, 2.0, 2.5], solver="ef_ipopt", seed=9 + ) + res2 = pest.profile_likelihood( + "theta_a", grid=[1.5, 2.0, 2.5], solver="ef_ipopt", seed=9 + ) + cols = [ + "theta_value", + "obj", + "status", + "success", + "theta__theta_a", + "theta__theta_b", + ] + self.assertTrue(res1["profiles"][cols].equals(res2["profiles"][cols])) + + def test_profile_warmstart_neighbor_values_used(self): + pest = _build_two_theta_estimator() + call_inits = [] + + def fake_q_opt(*args, **kwargs): + init = dict(kwargs["theta_vals"]) + fixed = kwargs["fixed_theta_values"] + call_inits.append(init) + return ( + float((fixed["theta_a"] - 2.0) ** 2), + {"theta_a": fixed["theta_a"], "theta_b": fixed["theta_a"] + 10.0}, + pyo.TerminationCondition.optimal, + ) + + with patch.object(pest, "_Q_opt", side_effect=fake_q_opt): + pest.profile_likelihood( + "theta_a", + grid=[2.0, 2.1, 2.2], + theta_hat={"theta_a": 2.0, "theta_b": 1.0}, + obj_hat=0.0, + warmstart="neighbor", + ) + + self.assertGreaterEqual(len(call_inits), 2) + self.assertAlmostEqual(call_inits[1]["theta_b"], 12.0, places=12) + + def test_profile_failure_recorded_continue(self): + pest = _build_two_theta_estimator() + + def fake_q_opt(*args, **kwargs): + a = kwargs["fixed_theta_values"]["theta_a"] + if abs(a - 2.0) < 1e-12: + raise RuntimeError("boom") + return ( + 1.0 + abs(a), + {"theta_a": a, "theta_b": 0.0}, + pyo.TerminationCondition.optimal, + ) + + with patch.object(pest, "_Q_opt", side_effect=fake_q_opt): + res = pest.profile_likelihood( + "theta_a", + grid=[1.9, 2.0, 2.1], + theta_hat={"theta_a": 2.0, "theta_b": 0.0}, + obj_hat=1.0, + ) + + prof = res["profiles"].sort_values("theta_value").reset_index(drop=True) + self.assertEqual(len(prof), 3) + self.assertIn("exception", str(prof.loc[1, "status"])) + self.assertFalse(bool(prof.loc[1, "success"])) + + def test_profile_all_failures_returns_structure(self): + pest = _build_two_theta_estimator() + + with patch.object(pest, "_Q_opt", side_effect=RuntimeError("all failed")): + res = pest.profile_likelihood( + "theta_a", + grid=[1.9, 2.0, 2.1], + theta_hat={"theta_a": 2.0, "theta_b": 0.0}, + obj_hat=1.0, + ) + + prof = res["profiles"] + self.assertEqual(len(prof), 3) + self.assertFalse(prof["success"].any()) + self.assertTrue(prof["status"].astype(str).str.contains("exception").all()) + + def test_profile_user_grid_preserved(self): + pest = _build_two_theta_estimator() + + with patch.object( + pest, + "_Q_opt", + side_effect=lambda *args, **kwargs: ( + 1.0, + {"theta_a": kwargs["fixed_theta_values"]["theta_a"], "theta_b": 0.0}, + pyo.TerminationCondition.optimal, + ), + ): + res = pest.profile_likelihood( + "theta_a", + grid=[1.2, 2.4, 2.0], + theta_hat={"theta_a": 2.0, "theta_b": 0.0}, + obj_hat=1.0, + ) + + attempted = sorted(res["profiles"]["theta_value"].tolist()) + self.assertEqual(attempted, [1.2, 2.0, 2.4]) + + def test_profile_auto_grid_includes_theta_hat(self): + pest = _build_two_theta_estimator() + grid = pest._build_profile_grid( + profiled_theta="theta_a", + grid=[1.0, 1.5, 2.5], + n_grid=5, + theta_hat={"theta_a": 2.0, "theta_b": 0.0}, + ) + self.assertIn(2.0, set(np.round(grid, 12))) + + def test_profile_result_columns_schema(self): + pest = _build_two_theta_estimator() + + with patch.object( + pest, + "_Q_opt", + side_effect=lambda *args, **kwargs: ( + 1.0, + {"theta_a": kwargs["fixed_theta_values"]["theta_a"], "theta_b": 0.0}, + pyo.TerminationCondition.optimal, + ), + ): + res = pest.profile_likelihood( + "theta_a", + grid=[1.9, 2.0], + theta_hat={"theta_a": 2.0, "theta_b": 0.0}, + obj_hat=1.0, + ) + prof = res["profiles"] + self.assertTrue( + set( + [ + "profiled_theta", + "theta_value", + "obj", + "delta_obj", + "lr_stat", + "status", + "success", + "solve_time", + ] + ).issubset(prof.columns) + ) + + @unittest.skipIf(not ipopt_available, "The 'ipopt' solver is not available") + def test_profile_baseline_from_multistart(self): + pest = _build_two_theta_estimator() + res = pest.profile_likelihood( + profiled_theta="theta_a", + n_grid=5, + use_multistart_for_baseline=True, + baseline_multistart_kwargs={ + "n_restarts": 3, + "multistart_sampling_method": "uniform_random", + "seed": 7, + }, + ) + self.assertIn("baseline", res) + self.assertIn("profiles", res) + self.assertTrue(np.isfinite(res["baseline"]["obj_hat"])) + self.assertGreaterEqual(res["profiles"].shape[0], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyomo/contrib/parmest/utils/create_ef.py b/pyomo/contrib/parmest/utils/create_ef.py index 56048fced10..17336114668 100644 --- a/pyomo/contrib/parmest/utils/create_ef.py +++ b/pyomo/contrib/parmest/utils/create_ef.py @@ -21,6 +21,7 @@ from pyomo.core import Objective +# File no longer used in parmest; retained for possible future use. def get_objs(scenario_instance): """return the list of objective functions for scenario_instance""" scenario_objs = scenario_instance.component_data_objects( diff --git a/pyomo/contrib/parmest/utils/mpi_utils.py b/pyomo/contrib/parmest/utils/mpi_utils.py index 82376062c45..30560a6a77d 100644 --- a/pyomo/contrib/parmest/utils/mpi_utils.py +++ b/pyomo/contrib/parmest/utils/mpi_utils.py @@ -10,6 +10,8 @@ from collections import OrderedDict import importlib +# ParallelTaskManager is used, MPI Interface is not. + """ This module is a collection of classes that provide a friendlier interface to MPI (through mpi4py). They help