Merge pull request #691 from stan-dev/release/1.2-prep

Prep for release v1.2.0

Author: WardBrian

.github/workflows/main.yml

@@ -110,7 +110,9 @@ jobs:
         run: |
           install_cmdstan -h
           install_cxx_toolchain -h
-          python -m cmdstanpy.install_cmdstan --compiler --version ${{ needs.get-cmdstan-version.outputs.version }} --cores 2
+          $env:TBB_CXXFLAGS = "-D_UCRT"
+          python -m cmdstanpy.install_cmdstan --version ${{ needs.get-cmdstan-version.outputs.version }} --cores 2
+          echo "TBB_CXXFLAGS+=-D_UCRT" >> ~/.cmdstan/cmdstan-${{ needs.get-cmdstan-version.outputs.version }}/make/local
 
       - name: Run tests
         run: |

cmdstanpy/compilation.py

@@ -30,6 +30,7 @@
     'warn-pedantic',
 ]
 
+# TODO(2.0): remove
 STANC_DEPRECATED_OPTS = {
     'allow_undefined': 'allow-undefined',
     'include_paths': 'include-paths',
@@ -59,6 +60,7 @@
 OptionalPath = Union[str, os.PathLike, None]
 
 
+# TODO(2.0): can remove add function and other logic
 class CompilerOptions:
     """
     User-specified flags for stanc and C++ compiler.

cmdstanpy/model.py

@@ -434,6 +434,7 @@ def code(self) -> Optional[str]:
             )
         return code
 
+    # TODO(2.0): remove
     def compile(
         self,
         force: bool = False,
@@ -1162,17 +1163,21 @@ def sample(
             if not runset._check_retcodes():
                 msg = (
                     f'Error during sampling:\n{errors}\n'
-                    + f'Command and output files:\n{repr(runset)}\n'
-                    + 'Consider re-running with show_console=True if the above'
-                    + ' output is unclear!'
+                    f'Command and output files:\n{repr(runset)}'
                 )
+                if not show_console:
+                    msg += (
+                        '\nConsider re-running with show_console=True if the'
+                        ' above output is unclear!'
+                    )
                 raise RuntimeError(msg)
             if errors:
-                msg = (
-                    f'Non-fatal error during sampling:\n{errors}\n'
-                    + 'Consider re-running with show_console=True if the above'
-                    + ' output is unclear!'
-                )
+                msg = f'Non-fatal error during sampling:\n{errors}'
+                if not show_console:
+                    msg += (
+                        '\nConsider re-running with show_console=True if the'
+                        ' above output is unclear!'
+                    )
                 get_logger().warning(msg)
 
             mcmc = CmdStanMCMC(runset)
@@ -1256,6 +1261,7 @@ def generate_quantities(
 
         :return: CmdStanGQ object
         """
+        # TODO(2.0): remove
         if mcmc_sample is not None:
             if previous_fit:
                 raise ValueError(
@@ -1359,10 +1365,13 @@ def generate_quantities(
             if errors:
                 msg = (
                     f'Error during generate_quantities:\n{errors}\n'
-                    + f'Command and output files:\n{repr(runset)}\n'
-                    + 'Consider re-running with show_console=True if the above'
-                    + ' output is unclear!'
+                    f'Command and output files:\n{repr(runset)}'
                 )
+                if not show_console:
+                    msg += (
+                        '\nConsider re-running with show_console=True if the'
+                        ' above output is unclear!'
+                    )
                 raise RuntimeError(msg)
             quantities = CmdStanGQ(runset=runset, previous_fit=fit_object)
         return quantities
@@ -1490,6 +1499,7 @@ def variational(
 
         :return: CmdStanVB object
         """
+        # TODO(2.0): remove
         if output_samples is not None:
             if draws is not None:
                 raise ValueError(
@@ -1627,7 +1637,7 @@ def pathfinder(
         :param draws: Number of approximate draws to return.
 
         :param num_single_draws: Number of draws each single-pathfinder will
-            draw. By default, this is set to be equal to draws.
+            draw.
             If ``num_paths`` is 1, only one of this and ``draws`` should be
             used.
 
@@ -1722,13 +1732,14 @@ def pathfinder(
                 "before 2.33"
             )
 
-        if num_single_draws is None:
-            num_single_draws = draws
-        elif num_paths == 1 and draws is not None and num_single_draws != draws:
-            raise ValueError(
-                "Cannot specify both 'draws' and 'num_single_draws'"
-                " when 'num_paths' is 1"
-            )
+        if num_paths == 1:
+            if num_single_draws is None:
+                num_single_draws = draws
+            if draws is not None and num_single_draws != draws:
+                raise ValueError(
+                    "Cannot specify both 'draws' and 'num_single_draws'"
+                    " when 'num_paths' is 1"
+                )
 
         pathfinder_args = PathfinderArgs(
             init_alpha=init_alpha,

cmdstanpy/stanfit/gq.py

@@ -172,6 +172,7 @@ def draws(
         self,
         *,
         inc_warmup: bool = False,
+        inc_iterations: bool = False,
         concat_chains: bool = False,
         inc_sample: bool = False,
     ) -> np.ndarray:
@@ -204,6 +205,7 @@ def draws(
         CmdStanMCMC.draws
         """
         self._assemble_generated_quantities()
+        inc_warmup |= inc_iterations
         if inc_warmup:
             if (
                 isinstance(self.previous_fit, CmdStanMCMC)
@@ -522,11 +524,7 @@ def draws_xr(
             'chain', 'draw', ...
         )
 
-    def stan_variable(
-        self,
-        var: str,
-        inc_warmup: bool = False,
-    ) -> np.ndarray:
+    def stan_variable(self, var: str, **kwargs: bool) -> np.ndarray:
         """
         Return a numpy.ndarray which contains the set of draws
         for the named Stan program variable.  Flattens the chains,
@@ -557,9 +555,9 @@ def stan_variable(
 
         :param var: variable name
 
-        :param inc_warmup: When ``True`` and the warmup draws are present in
-            the MCMC sample, then the warmup draws are included.
-            Default value is ``False``.
+        :param kwargs: Additional keyword arguments are passed to the underlying
+            fit's ``stan_variable`` method if the variable is not a generated
+            quantity.
 
         See Also
         --------
@@ -579,36 +577,30 @@ def stan_variable(
                 + ", ".join(model_var_names | gq_var_names)
             )
         if var not in gq_var_names:
-            if isinstance(self.previous_fit, CmdStanMCMC):
-                return self.previous_fit.stan_variable(
-                    var, inc_warmup=inc_warmup
-                )
-            elif isinstance(self.previous_fit, CmdStanMLE):
-                return np.atleast_1d(  # type: ignore
-                    self.previous_fit.stan_variable(
-                        var, inc_iterations=inc_warmup
-                    )
-                )
-            else:
-                return np.atleast_1d(  # type: ignore
-                    self.previous_fit.stan_variable(var)
-                )
+            # TODO(2.0) atleast1d may not be needed
+            return np.atleast_1d(  # type: ignore
+                self.previous_fit.stan_variable(var, **kwargs)
+            )
+
         # is gq variable
         self._assemble_generated_quantities()
 
-        draw1, _ = self._draws_start(inc_warmup)
+        draw1, _ = self._draws_start(
+            inc_warmup=kwargs.get('inc_warmup', False)
+            or kwargs.get('inc_iterations', False)
+        )
         draws = flatten_chains(self._draws[draw1:])
         out: np.ndarray = self._metadata.stan_vars[var].extract_reshape(draws)
         return out
 
-    def stan_variables(self, inc_warmup: bool = False) -> Dict[str, np.ndarray]:
+    def stan_variables(self, **kwargs: bool) -> Dict[str, np.ndarray]:
         """
         Return a dictionary mapping Stan program variables names
         to the corresponding numpy.ndarray containing the inferred values.
 
-        :param inc_warmup: When ``True`` and the warmup draws are present in
-            the MCMC sample, then the warmup draws are included.
-            Default value is ``False``
+        :param kwargs: Additional keyword arguments are passed to the underlying
+            fit's ``stan_variable`` method if the variable is not a generated
+            quantity.
 
         See Also
         --------
@@ -623,10 +615,10 @@ def stan_variables(self, inc_warmup: bool = False) -> Dict[str, np.ndarray]:
         sample_var_names = self.previous_fit._metadata.stan_vars.keys()
         gq_var_names = self._metadata.stan_vars.keys()
         for name in gq_var_names:
-            result[name] = self.stan_variable(name, inc_warmup)
+            result[name] = self.stan_variable(name, **kwargs)
         for name in sample_var_names:
             if name not in gq_var_names:
-                result[name] = self.stan_variable(name, inc_warmup)
+                result[name] = self.stan_variable(name, **kwargs)
         return result
 
     def _assemble_generated_quantities(self) -> None:
@@ -733,6 +725,7 @@ def save_csvfiles(self, dir: Optional[str] = None) -> None:
         """
         self.runset.save_csvfiles(dir)
 
+    # TODO(2.0): remove
     @property
     def mcmc_sample(self) -> Union[CmdStanMCMC, CmdStanMLE, CmdStanVB]:
         get_logger().warning(

cmdstanpy/stanfit/mle.py

@@ -169,7 +169,7 @@ def stan_variable(
         *,
         inc_iterations: bool = False,
         warn: bool = True,
-    ) -> np.ndarray:
+    ) -> Union[np.ndarray, float]:
         """
         Return a numpy.ndarray which contains the estimates for the
         for the named Stan program variable where the dimensions of the
@@ -217,6 +217,14 @@ def stan_variable(
             out: np.ndarray = self._metadata.stan_vars[var].extract_reshape(
                 data
             )
+            # TODO(2.0) remove
+            if out.shape == () or out.shape == (1,):
+                get_logger().warning(
+                    "The default behavior of CmdStanMLE.stan_variable() "
+                    "will change in a future release to always return a "
+                    "numpy.ndarray, even for scalar variables."
+                )
+                return out.item()  # type: ignore
             return out
         except KeyError:
             # pylint: disable=raise-missing-from
@@ -228,7 +236,7 @@ def stan_variable(
 
     def stan_variables(
         self, inc_iterations: bool = False
-    ) -> Dict[str, np.ndarray]:
+    ) -> Dict[str, Union[np.ndarray, float]]:
         """
         Return a dictionary mapping Stan program variables names
         to the corresponding numpy.ndarray containing the inferred values.

cmdstanpy/stanfit/vb.py

@@ -8,6 +8,7 @@
 
 from cmdstanpy.cmdstan_args import Method
 from cmdstanpy.utils import scan_variational_csv
+from cmdstanpy.utils.logging import get_logger
 
 from .metadata import InferenceMetadata
 from .runset import RunSet
@@ -113,7 +114,9 @@ def metadata(self) -> InferenceMetadata:
         """
         return self._metadata
 
-    def stan_variable(self, var: str) -> np.ndarray:
+    def stan_variable(
+        self, var: str, *, mean: Optional[bool] = None
+    ) -> Union[np.ndarray, float]:
         """
         Return a numpy.ndarray which contains the estimates for the
         for the named Stan program variable where the dimensions of the
@@ -135,6 +138,10 @@ def stan_variable(self, var: str) -> np.ndarray:
 
         :param var: variable name
 
+        :param mean: if True, return the variational mean. Otherwise,
+            return the variational sample.  The default behavior will
+            change in a future release to return the variational sample.
+
         See Also
         --------
         CmdStanVB.stan_variables
@@ -144,10 +151,35 @@ def stan_variable(self, var: str) -> np.ndarray:
         CmdStanGQ.stan_variable
         CmdStanLaplace.stan_variable
         """
+        # TODO(2.0): remove None case, make default `False`
+        if mean is None:
+            get_logger().warning(
+                "The default behavior of CmdStanVB.stan_variable() "
+                "will change in a future release to return the "
+                "variational sample, rather than the mean.\n"
+                "To maintain the current behavior, pass the argument "
+                "mean=True"
+            )
+            mean = True
+        if mean:
+            draws = self._variational_mean
+        else:
+            draws = self._variational_sample
+
         try:
             out: np.ndarray = self._metadata.stan_vars[var].extract_reshape(
-                self._variational_sample
+                draws
             )
+            # TODO(2.0): remove
+            if out.shape == () or out.shape == (1,):
+                if mean:
+                    get_logger().warning(
+                        "The default behavior of "
+                        "CmdStanVB.stan_variable(mean=True) will change in a "
+                        "future release to always return a numpy.ndarray, even "
+                        "for scalar variables."
+                    )
+                return out.item()  # type: ignore
             return out
         except KeyError:
             # pylint: disable=raise-missing-from
@@ -157,7 +189,9 @@ def stan_variable(self, var: str) -> np.ndarray:
                 + ", ".join(self._metadata.stan_vars.keys())
             )
 
-    def stan_variables(self) -> Dict[str, np.ndarray]:
+    def stan_variables(
+        self, *, mean: Optional[bool] = None
+    ) -> Dict[str, Union[np.ndarray, float]]:
         """
         Return a dictionary mapping Stan program variables names
         to the corresponding numpy.ndarray containing the inferred values.
@@ -173,7 +207,7 @@ def stan_variables(self) -> Dict[str, np.ndarray]:
         """
         result = {}
         for name in self._metadata.stan_vars:
-            result[name] = self.stan_variable(name)
+            result[name] = self.stan_variable(name, mean=mean)
         return result
 
     @property

docsrc/changes.rst

@@ -7,6 +7,36 @@ What's New
 
 For full changes, see the `Releases page <https://github.com/stan-dev/cmdstanpy/releases>`_ on GitHub.
 
+CmdStanPy 1.2.0
+---------------
+- **New functionality**
+
+  - The Pathfinder algorithm (available in CmdStan 2.33+) is now availble as :meth:`CmdStanModel.pathfinder`.
+  - Laplace approximations (available in CmdStan 2.32+) are now available as :meth:`CmdStanModel.laplace_sample`.
+  - The :meth:`CmdStanModel.optimize` method now supports the ``jacobian`` boolean argument to enable change-of-variables adjustments.
+    When enabled, the Maximum a posteriori estimate (MAP) is returned, rather than the MLE.
+  - The :func:`cmdstanpy.install_cmdstan` function and script can install development versions of CmdStan using the ``git:`` prefix in the version.
+
+- **Deprecations**
+  The next non-bugfix release of CmdStanPy will be version 2.0, which will remove all existing deprecations. Additional deprecations in this version:
+
+  - :class:`CmdStanModel` will *require* that it has a compiled executable after construction. The ``compile`` argument is deprecated,
+    (the ability to force recompilation is available under the argument ``force_compile``), and the ``compile()`` method is deprecated.
+    If you wish to compile Stan files independent of constructing a model, use :func:`cmdstanpy.compile_stan_file`.
+  - :meth:`CmdStanMLE.stan_variable` will begin returning a :class:`np.ndarray` in all cases, as opposed to the current behavior where sometimes a float is returned.
+  - :meth:`CmdStanVB.stan_variables` will return the _draws_ from the approximate posterior, rather than the optimized mean.
+    A new argument, ``mean``, can be set to True to return the mean instead. Additionally, a :class:`np.ndarray` will be returned in all cases starting in the next version.
+  - :meth:`CmdStanModel.variational` argument ``output_samples`` will has been renamed to ``draws``.
+
+- **Other changes**
+
+  - A list of dictionaries is now allowed as the ``inits`` argument to :meth:`CmdStanModel.sample`.
+  - :func:`cmdstanpy.install_cmdstan` correctly fetches the CmdStan version for ppc64el machines.
+  - The documentation on how to use external C++ code was updated.
+  - Various other bug fixes.
+
+.. note::
+    The minimum supported version for CmdStanPy is now Python 3.8.
 
 CmdStanPy 1.1.0
 ---------------