diff --git a/doc/Makefile b/doc/Makefile index 072cf38e..a65b4342 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -12,7 +12,12 @@ BUILDDIR = _build help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: help Makefile +.PHONY: help clean Makefile + +# Clean target that removes build directory +clean: + @echo "Removing build directory..." + rm -rf "$(BUILDDIR)" # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/doc/api.rst b/doc/api.rst index 44b9f05a..6011aa81 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -121,21 +121,32 @@ IO functions model.Model.to_netcdf io.read_netcdf +Solver utilities +================= + +.. autosummary:: + :toctree: generated/ + + solvers.available_solvers + solvers.quadratic_solvers + solvers.Solver + + Solvers -======== +======= .. autosummary:: :toctree: generated/ - solvers.run_cbc - solvers.run_glpk - solvers.run_highs - solvers.run_cplex - solvers.run_gurobi - solvers.run_xpress - solvers.run_mosek - solvers.run_mindopt - solvers.run_copt + solvers.CBC + solvers.Cplex + solvers.GLPK + solvers.Gurobi + solvers.Highs + solvers.Mosek + solvers.SCIP + solvers.Xpress + Solving ======== diff --git a/doc/conf.py b/doc/conf.py index eadf907c..e68f036c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -90,7 +90,15 @@ """ -nbsphinx_allow_errors = False +nbsphinx_allow_errors = True +nbsphinx_execute = "auto" +nbsphinx_execute_arguments = [ + "--InlineBackend.figure_formats={'svg', 'pdf'}", + "--InlineBackend.rc={'figure.dpi': 96}", +] + +# Exclude notebooks that require credentials or special setup +nbsphinx_execute_never = ["**/solve-on-oetc*"] # -- Options for HTML output ------------------------------------------------- diff --git a/doc/index.rst b/doc/index.rst index 2591021b..f05f89f3 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -114,6 +114,7 @@ This package is published under MIT license. transport-tutorial infeasible-model solve-on-remote + solve-on-oetc migrating-from-pyomo gurobi-double-logging diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 97f1abde..22c0ffee 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -2,7 +2,6 @@ Release Notes ============= Upcoming Version ----------------- * Replace pandas-based LP file writing with polars implementation for significantly improved performance on large models * Consolidate "lp" and "lp-polars" io_api options - both now use the optimized polars backend @@ -37,7 +36,7 @@ Version 0.5.4 **Bug Fixes** * Remove default highs log file when `log_fn=None` and `io_api="direct"`. This caused `log_file` in -`solver_options` to be ignored. + `solver_options` to be ignored. * Fix the parsing of solutions returned by the CBC solver when setting a MIP duality gap tolerance. * Improve the mapping of termination conditions for the SCIP solver @@ -65,8 +64,8 @@ Version 0.5.2 **Bug Fixes** * Fix the multiplication with of zero dimensional numpy arrays with linopy objects. -This is mainly affecting operations where single numerical items from pandas objects -are selected and used for multiplication. + This is mainly affecting operations where single numerical items from pandas objects + are selected and used for multiplication. Version 0.5.1 -------------- @@ -199,7 +198,7 @@ Version 0.3.9 * The constraint assignment with a `LinearExpression` and a constant value when using the pattern `model.add_constraints(lhs_with_constant, sign, rhs)` was fixed. Before, the constant value was not added to the right-hand-side properly which led to the wrong constraint behavior. This is fixed now. -* `nan`s in constants is now handled more consistently. These are ignored when in the addition of expressions (effectively filled by zero). In a future version, this might change to align the propagation of `nan`s with tools like numpy/pandas/xarray. +* ``nan`` s in constants is now handled more consistently. These are ignored when in the addition of expressions (effectively filled by zero). In a future version, this might change to align the propagation of ``nan`` s with tools like numpy/pandas/xarray. * Up to now the `rhs` argument in the `add_constraints` function was not supporting an expression as an input type. This is now added. diff --git a/doc/solve-on-oetc.nblink b/doc/solve-on-oetc.nblink new file mode 100644 index 00000000..ab7ed00c --- /dev/null +++ b/doc/solve-on-oetc.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/solve-on-oetc.ipynb" +} diff --git a/examples/creating-expressions.ipynb b/examples/creating-expressions.ipynb index c57ad94b..aafd8a09 100644 --- a/examples/creating-expressions.ipynb +++ b/examples/creating-expressions.ipynb @@ -160,10 +160,7 @@ "cell_type": "markdown", "id": "f7578221", "metadata": {}, - "source": [ - ".. important::\n", - " When combining variables or expression with dimensions of the same name and size, the first object will determine the coordinates of the resulting expression. For example: " - ] + "source": ".. important::\n\n\tWhen combining variables or expression with dimensions of the same name and size, the first object will determine the coordinates of the resulting expression. For example:" }, { "cell_type": "code", diff --git a/examples/creating-variables.ipynb b/examples/creating-variables.ipynb index 7527fa71..8e879348 100644 --- a/examples/creating-variables.ipynb +++ b/examples/creating-variables.ipynb @@ -467,13 +467,7 @@ "cell_type": "markdown", "id": "77e264e2", "metadata": {}, - "source": [ - ".. important::\n", - "\n", - " **New in version 0.3.6**\n", - "\n", - " As pandas objects always have indexes, the `coords` argument is not required and is ignored is passed. Before, it was used to overwrite the indexes of the pandas objects. A warning is raised if `coords` is passed and if these are not aligned with the pandas object. " - ] + "source": ".. important::\n\n **New in version 0.3.6**\n\n As pandas objects always have indexes, the `coords` argument is not required and is ignored is passed. Before, it was used to overwrite the indexes of the pandas objects. A warning is raised if `coords` is passed and if these are not aligned with the pandas object." }, { "cell_type": "code", diff --git a/examples/solve-on-oetc.ipynb b/examples/solve-on-oetc.ipynb new file mode 100644 index 00000000..c940fd11 --- /dev/null +++ b/examples/solve-on-oetc.ipynb @@ -0,0 +1,363 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Solve on OETC (OET Cloud)\n", + "\n", + "This example demonstrates how to use linopy with OETC (OET Cloud) for cloud-based optimization solving. OETC is a cloud platform that provides scalable computing resources for optimization problems.\n", + "\n", + "## What you need to run this example:\n", + "\n", + "* A working installation of the required packages:\n", + " * `pip install google-cloud-storage requests`\n", + "* An OETC account with valid credentials (email and password)\n", + "* Access to OETC authentication and orchestrator servers\n", + "\n", + "## How OETC Cloud Solving Works\n", + "\n", + "The OETC integration follows this workflow:\n", + "\n", + "1. **Model Creation**: Define your optimization model locally using linopy\n", + "2. **Authentication**: Sign in to the OETC platform using your credentials\n", + "3. **File Upload**: Compress and upload your model to Google Cloud Storage\n", + "4. **Job Submission**: Submit a compute job to the OETC orchestrator\n", + "5. **Job Monitoring**: Wait for job completion with automatic status polling\n", + "6. **Solution Download**: Download and decompress the solved model\n", + "7. **Local Integration**: Load the solution back into your local model\n", + "\n", + "All of these steps are handled automatically by linopy's `OetcHandler`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create a Model\n", + "\n", + "First, let's create an optimization model that we want to solve on OETC:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from numpy import arange\n", + "from xarray import DataArray\n", + "\n", + "from linopy import Model\n", + "\n", + "# Create a medium-sized optimization problem\n", + "N = 50\n", + "m = Model()\n", + "\n", + "# Define decision variables with coordinates\n", + "coords = [arange(N), arange(N)]\n", + "x = m.add_variables(coords=coords, name=\"x\", lower=0)\n", + "y = m.add_variables(coords=coords, name=\"y\", lower=0)\n", + "\n", + "# Add constraints\n", + "m.add_constraints(x - y >= DataArray(arange(N)), name=\"constraint1\")\n", + "m.add_constraints(x + y >= DataArray(arange(N) * 0.5), name=\"constraint2\")\n", + "m.add_constraints(x <= DataArray(arange(N) + 10), name=\"upper_bounds\")\n", + "\n", + "# Set objective function\n", + "m.add_objective((2 * x + y).sum())\n", + "\n", + "print(\n", + " f\"Model created with {len(m.variables)} variable groups and {len(m.constraints)} constraint groups\"\n", + ")\n", + "m" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configure OETC Settings\n", + "\n", + "Next, we need to configure the OETC settings including credentials and compute requirements:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Configure your OETC credentials\n", + "# IMPORTANT: Never hardcode credentials in production code!\n", + "# Use environment variables or secure credential management\n", + "import os\n", + "\n", + "from linopy.remote.oetc import (\n", + " ComputeProvider,\n", + " OetcCredentials,\n", + " OetcHandler,\n", + " OetcSettings,\n", + ")\n", + "\n", + "credentials = OetcCredentials(\n", + " email=os.getenv(\"OETC_EMAIL\", \"your-email@example.com\"),\n", + " password=os.getenv(\"OETC_PASSWORD\", \"your-password\"),\n", + ")\n", + "\n", + "# Configure OETC settings\n", + "settings = OetcSettings(\n", + " credentials=credentials,\n", + " name=\"linopy-example-job\",\n", + " authentication_server_url=\"https://auth.oetcloud.com\", # Replace with actual URL\n", + " orchestrator_server_url=\"https://orchestrator.oetcloud.com\", # Replace with actual URL\n", + " compute_provider=ComputeProvider.GCP,\n", + " cpu_cores=4, # Number of CPU cores to allocate\n", + " disk_space_gb=20, # Disk space in GB\n", + " delete_worker_on_error=False, # Keep worker for debugging if job fails\n", + ")\n", + "\n", + "print(\"OETC settings configured successfully\")\n", + "print(f\"Solver: {settings.solver}\")\n", + "print(f\"CPU cores: {settings.cpu_cores}\")\n", + "print(f\"Disk space: {settings.disk_space_gb} GB\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize OETC Handler\n", + "\n", + "The `OetcHandler` manages the entire cloud solving process:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the OETC handler\n", + "# This will authenticate with OETC and fetch cloud provider credentials\n", + "oetc_handler = OetcHandler(settings)\n", + "\n", + "print(\"OETC handler initialized successfully\")\n", + "print(f\"Authentication token expires at: {oetc_handler.jwt.expires_at}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Solve the Model on OETC\n", + "\n", + "Now we can solve our model on the OETC cloud platform. The `OetcHandler` is passed to the model's `solve()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Solve the model on OETC\n", + "# This will upload the model, submit a job, wait for completion, and download the solution\n", + "import time\n", + "\n", + "print(\"Starting cloud solving process...\")\n", + "start_time = time.time()\n", + "\n", + "try:\n", + " status, termination_condition = m.solve(remote=oetc_handler)\n", + "\n", + " end_time = time.time()\n", + " total_time = end_time - start_time\n", + "\n", + " print(f\"\\nSolving completed in {total_time:.2f} seconds\")\n", + " print(f\"Status: {status}\")\n", + " print(f\"Termination condition: {termination_condition}\")\n", + " print(f\"Objective value: {m.objective.value:.4f}\")\n", + "\n", + "except Exception as e:\n", + " print(f\"Error during solving: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Examine the Solution\n", + "\n", + "Let's examine the solution returned from OETC:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Display solution summary\n", + "print(f\"Model status: {m.status}\")\n", + "print(f\"Objective value: {m.objective.value}\")\n", + "print(f\"Number of variables: {m.solution.sizes}\")\n", + "\n", + "# Show a subset of the solution\n", + "print(\"\\nSample of solution values:\")\n", + "print(\"x values (first 5x5):\")\n", + "print(m.solution[\"x\"].isel(dim_0=slice(0, 5), dim_1=slice(0, 5)).values)\n", + "\n", + "print(\"\\ny values (first 5x5):\")\n", + "print(m.solution[\"y\"].isel(dim_0=slice(0, 5), dim_1=slice(0, 5)).values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Advanced OETC Configuration\n", + "\n", + "### Solver Options\n", + "\n", + "You can pass solver-specific options through the `solver_options` parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example with advanced solver options\n", + "advanced_settings = OetcSettings(\n", + " credentials=credentials,\n", + " name=\"advanced-linopy-job\",\n", + " authentication_server_url=\"https://auth.oetcloud.com\",\n", + " orchestrator_server_url=\"https://orchestrator.oetcloud.com\",\n", + " solver=\"gurobi\", # Using Gurobi solver\n", + " solver_options={\n", + " \"TimeLimit\": 600, # 10 minutes\n", + " \"MIPGap\": 0.01, # 1% optimality gap\n", + " \"Threads\": 4, # Use 4 threads\n", + " \"OutputFlag\": 1, # Enable solver output\n", + " },\n", + " cpu_cores=8, # More CPU cores for larger problems\n", + " disk_space_gb=50, # More disk space\n", + ")\n", + "\n", + "print(\"Advanced OETC settings:\")\n", + "print(f\"Solver: {advanced_settings.solver}\")\n", + "print(f\"Solver options: {advanced_settings.solver_options}\")\n", + "print(f\"CPU cores: {advanced_settings.cpu_cores}\")\n", + "print(f\"Disk space: {advanced_settings.disk_space_gb} GB\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Error Handling and Debugging\n", + "\n", + "When working with cloud solving, it's important to handle potential errors gracefully:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def solve_with_error_handling(model, oetc_handler, max_retries=3):\n", + " \"\"\"Solve model with error handling and retries\"\"\"\n", + "\n", + " for attempt in range(max_retries):\n", + " try:\n", + " print(f\"Solving attempt {attempt + 1}/{max_retries}...\")\n", + " status, termination = model.solve(remote=oetc_handler)\n", + "\n", + " if status == \"ok\":\n", + " print(\"Solving successful!\")\n", + " return status, termination\n", + " else:\n", + " print(f\"Solving returned status: {status}\")\n", + "\n", + " except Exception as e:\n", + " print(f\"Attempt {attempt + 1} failed: {e}\")\n", + "\n", + " if attempt < max_retries - 1:\n", + " print(\"Retrying in 30 seconds...\")\n", + " time.sleep(30)\n", + " else:\n", + " print(\"All attempts failed\")\n", + " raise\n", + "\n", + " return None, None\n", + "\n", + "\n", + "# Example usage (commented out to avoid actual execution)\n", + "# status, termination = solve_with_error_handling(m, oetc_handler)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Security Best Practices\n", + "\n", + "When using OETC in production:\n", + "\n", + "1. **Never hardcode credentials**: Use environment variables or secure credential stores\n", + "2. **Use token expiration**: The OETC handler automatically manages token expiration\n", + "3. **Validate inputs**: Ensure your model data doesn't contain sensitive information\n", + "4. **Monitor costs**: Cloud computing resources have associated costs\n", + "5. **Clean up resources**: Set `delete_worker_on_error=True` for automatic cleanup\n", + "\n", + "## Comparison with SSH Remote Solving\n", + "\n", + "| Feature | OETC Cloud | SSH Remote |\n", + "|---------|------------|------------|\n", + "| Setup | Account registration | Server access required |\n", + "| Scalability | Auto-scaling | Fixed server resources |\n", + "| Maintenance | Managed service | Self-managed |\n", + "| Cost | Pay-per-use | Infrastructure costs |\n", + "| Security | Enterprise-grade | Self-managed |\n", + "| Solver Licenses | Included | User-provided |\n", + "\n", + "Choose OETC for:\n", + "- Large-scale problems requiring significant compute resources\n", + "- Temporary or intermittent optimization needs\n", + "- Teams without dedicated infrastructure\n", + "- Access to premium solvers without license management\n", + "\n", + "Choose SSH remote for:\n", + "- Existing infrastructure with optimization solvers\n", + "- Strict data governance requirements\n", + "- Consistent, long-running optimization workloads\n", + "- Full control over the solving environment" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/solve-on-remote.ipynb b/examples/solve-on-remote.ipynb index 763ac438..16e01b41 100644 --- a/examples/solve-on-remote.ipynb +++ b/examples/solve-on-remote.ipynb @@ -4,26 +4,12 @@ "cell_type": "markdown", "id": "4db583af", "metadata": {}, - "source": [ - "# Solve on a Server\n", - "\n", - "In this example, we explain how linopy can handle model optimization on remote machines. \n", - "What you need in order to run the example:\n", - "* a running installation of paramiko on your local machine (use `pip install paramiko` for example)\n", - "* a remote server with an working installation of linopy, e.g. in a `conda` environment.\n", - "* a working ssh access to that machine \n", - "\n", + "source": ["# Remote Solving with SSH", "\n", + "This example demonstrates how linopy can solve optimization models on remote machines using SSH connections. This is one of two remote solving options available in linopy", "\n", - "The basic idea that the workflow follows consists of the following steps most of which linopy takes over for you:\n", - "\n", - "1. define a model on the local machine\n", - "2. save the model on the remote machine\n", - "3. load, solve and write out the model, all on the remote machine\n", - "4. copy the solved model to the local machine\n", - "5. load the solved model on the local machine\n", - "\n", - "Therefore the initialization process happens locally and the solving remotely. " - ] + "1. **SSH Remote Solving** (this example) - Connect to your own servers via SSH\\n2. **OETC Cloud Solving** - Use cloud-based optimization services (see `solve-on-oetc.ipynb`)", + "\n\n", + "## SSH Remote Solving\\n\\nSSH remote solving is ideal when you have:\\n* Access to dedicated servers with optimization solvers installed\\n* Full control over the computing environment\\n* Existing infrastructure for optimization workloads\\n\\n## What you need for SSH remote solving:\\n* A running installation of paramiko on your local machine (`pip install paramiko`)\\n* A remote server with a working installation of linopy (e.g., in a conda environment)\\n* SSH access to that machine\\n\\n## How SSH Remote Solving Works\\n\\nThe workflow consists of the following steps, most of which linopy handles automatically:\\n\\n1. Define a model on the local machine\\n2. Save the model on the remote machine via SSH\\n3. Load, solve and write out the model on the remote machine\\n4. Copy the solved model back to the local machine\\n5. Load the solved model on the local machine\\n\\nThe model initialization happens locally, while the actual solving happens remotely.\""] }, { "cell_type": "markdown", diff --git a/linopy/expressions.py b/linopy/expressions.py index 69095840..1078a080 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -1588,7 +1588,7 @@ def from_tuples( >>> y = m.add_variables(4, pd.Series([8, 10])) >>> expr = LinearExpression.from_tuples((10, x), (1, y), 1) - This is the same as calling ``10*x + y` + 1` but a bit more performant. + This is the same as calling ``10*x + y`` + 1 but a bit more performant. """ def process_one( diff --git a/linopy/model.py b/linopy/model.py index 149c2cc2..3982b84d 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -918,6 +918,7 @@ def linexpr( If args is a collection of coefficients-variables-tuples and constants, the resulting linear expression is built with the function LinearExpression.from_tuples. + * coefficients : int/float/array_like The coefficient(s) in the term, if the coefficients array contains dimensions which do not appear in diff --git a/pyproject.toml b/pyproject.toml index 18fdd8aa..b5105230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ docs = [ "sphinx_book_theme==1.1.3", "nbsphinx==0.9.4", "nbsphinx-link==1.3.0", + "docutils<0.21", + "numpy<2", "gurobipy==11.0.2", "ipykernel==6.29.5", "matplotlib==3.9.1",