diff --git a/.github/workflows/test-notebooks.yml b/.github/workflows/test-notebooks.yml index dfe025d2..2a5cec6e 100644 --- a/.github/workflows/test-notebooks.yml +++ b/.github/workflows/test-notebooks.yml @@ -40,8 +40,8 @@ jobs: # Skip notebooks that require credentials or special setup case "$name" in - solve-on-oetc.ipynb|solve-on-remote.ipynb) - echo "Skipping $name (requires credentials or special setup)" + remote-machines.ipynb) + echo "Skipping $name (requires credentials or remote machine)" continue ;; esac diff --git a/doc/api.rst b/doc/api.rst index f0afc322..3c59ef09 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -519,10 +519,19 @@ Solvers Remote solving ============== +Solve a model on a remote machine via SSH or on the OET Cloud (OETC). +See :doc:`remote-machines` for usage. + .. autosummary:: :toctree: generated/ + remote.SSH + remote.SshSettings + remote.Oetc + remote.OetcSettings remote.RemoteHandler + remote.OetcHandler + remote.OetcCredentials Solver status and result types diff --git a/doc/index.rst b/doc/index.rst index 39846607..a31d645a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -136,8 +136,7 @@ This package is published under MIT license. :maxdepth: 2 :caption: Solving - solve-on-remote - solve-on-oetc + remote-machines gpu-acceleration .. toctree:: diff --git a/doc/release_notes.rst b/doc/release_notes.rst index e5b7033f..7224e391 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -46,9 +46,39 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y * Xpress now supports ``io_api="direct"``: the linopy model is loaded via the native ``loadproblem`` array API instead of being serialised through an LP/MPS file, with SOS constraints attached in-place. Adds ``model.to_xpress()`` matching the existing ``to_gurobipy`` / ``to_highspy`` / ``to_mosek`` helpers. * Writing the solution back to the model after solving is faster: it no longer rebuilds the constraint matrix, and now uses positional (rather than label-based) indexing — roughly 2× faster overall. +*Remote solves* + +* Pass ``remote=`` to ``Model.solve`` to run the solver on a remote worker: + + .. code-block:: python + + m.solve("gurobi", remote=OetcSettings(...), Method=2) + m.solve("highs", remote=SshSettings(hostname=...), presolve="on") + + ``solver_name`` and ``**solver_options`` work the same as for local solves; ``remote=`` selects *where* to run. After the call, ``model.remote`` holds the remote instance (mirrors :attr:`Model.solver`). +* ``SshSettings.setup_commands: list[str]`` — shell commands run on the remote before the solve, e.g. ``setup_commands=["conda activate linopy-env"]``. + **Deprecations** * ``Solver.solve_problem``, ``Solver.solve_problem_from_model``, and ``Solver.solve_problem_from_file`` still work but emit a ``DeprecationWarning``. Use ``Solver.from_name(...).solve()`` (or simply ``model.solve(...)``) instead. They will be removed in a future release. +* ``linopy.remote.OetcHandler`` and ``linopy.remote.RemoteHandler`` are deprecated. Construction emits a ``DeprecationWarning``; the ``solve_on_oetc`` / ``solve_on_remote`` return contracts are unchanged. Migrate: + + .. code-block:: python + + # Before + handler = OetcHandler( + OetcSettings(credentials=OetcCredentials(email=..., password=...), ...) + ) + solved = handler.solve_on_oetc(m, TimeLimit=100) + + # After + m.solve( + "gurobi", remote=OetcSettings(email=..., password=..., ...), TimeLimit=100 + ) + + Passing an existing handler via ``Model.solve(remote=handler, ...)`` is also deprecated — pass the settings dataclass instead. +* ``linopy.remote.OetcCredentials`` is deprecated. Pass ``email`` and ``password`` directly to :class:`OetcSettings` instead of wrapping them. The ``OetcSettings(credentials=OetcCredentials(...))`` shape still works for one deprecation cycle and emits a ``DeprecationWarning``. +* :class:`linopy.remote.SSH` only exposes ``solve(model)``. For env activation use ``SshSettings.setup_commands``; for arbitrary remote shell commands, drop to :class:`RemoteHandler` (during deprecation) or paramiko directly. **Bug Fixes** @@ -60,6 +90,7 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y * ``available_solvers`` now lists all *installed* solvers, even ones without a working license. If you used it to decide "can I actually solve with X?", switch to ``linopy.licensed_solvers`` or ``SolverClass.license_status()``. * ``Model.solver_model`` and ``Model.solver_name`` are now read-only properties that delegate to ``model.solver``. You can't reassign them (only ``= None`` is allowed, which closes the solver), and ``solver_name`` is ``None`` before the first solve. * ``result.solution.primal`` and ``result.solution.dual`` are now ``numpy`` arrays indexed by linopy's integer labels (with ``NaN`` for slots without a value), instead of pandas Series keyed by variable/constraint name. If you accessed them by name, use ``model.variables[name].solution`` (or ``model.constraints[name].dual``) instead. +* The pip extra ``linopy[remote]`` has been renamed to ``linopy[ssh]`` to match what it installs (only ``paramiko``, for SSH transport — OETC has its own ``linopy[oetc]`` extra). ``linopy[remote]`` no longer exists; update your install commands. **Internal** diff --git a/doc/remote-machines.nblink b/doc/remote-machines.nblink new file mode 100644 index 00000000..f273fb0c --- /dev/null +++ b/doc/remote-machines.nblink @@ -0,0 +1,3 @@ +{ + "path": "../examples/remote-machines.ipynb" +} diff --git a/doc/solve-on-oetc.nblink b/doc/solve-on-oetc.nblink deleted file mode 100644 index ab7ed00c..00000000 --- a/doc/solve-on-oetc.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../examples/solve-on-oetc.ipynb" -} diff --git a/doc/solve-on-remote.nblink b/doc/solve-on-remote.nblink deleted file mode 100644 index 03be52c0..00000000 --- a/doc/solve-on-remote.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../examples/solve-on-remote.ipynb" -} diff --git a/doc/user-guide.rst b/doc/user-guide.rst index 8b7ee5bd..ce4549c3 100644 --- a/doc/user-guide.rst +++ b/doc/user-guide.rst @@ -53,8 +53,8 @@ Where to go next :doc:`piecewise-linear-constraints`, and the :doc:`testing-framework` for asserting structural properties of a model. -- **Solving** — :doc:`solve-on-remote` (SSH), - :doc:`solve-on-oetc` (OET Cloud), :doc:`gpu-acceleration` (cuPDLPx). +- **Solving** — :doc:`remote-machines` (SSH or OET Cloud), + :doc:`gpu-acceleration` (cuPDLPx). - **Troubleshooting** — :doc:`infeasible-model` (diagnosing infeasible problems), :doc:`gurobi-double-logging` (and other solver quirks). - **Reference** — the full :doc:`api` listing. diff --git a/examples/remote-machines.ipynb b/examples/remote-machines.ipynb new file mode 100644 index 00000000..00033483 --- /dev/null +++ b/examples/remote-machines.ipynb @@ -0,0 +1,238 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Remote machines\n", + "\n", + "linopy can ship your model to a remote machine, run a solver there, and pull the solved model back. Two transports are supported:\n", + "\n", + "- **SSH** — connect to a server you own (or have access to) over SSH.\n", + "- **OETC** — submit jobs to [OET Cloud](https://open-energy-transition.org/), a managed optimization service.\n", + "\n", + "Both share the same entry point on `Model.solve`:\n", + "\n", + "```python\n", + "m.solve(\"gurobi\", remote=, **solver_options)\n", + "```\n", + "\n", + "`solver_name` and `**solver_options` work exactly like a local solve; `remote=` selects *where* to run. After the call, `model.remote` holds the transport instance for post-solve introspection (mirrors `model.solver`)." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "> **Note:** This notebook is not executed during the documentation build — it requires either SSH access to a remote server or OETC credentials." + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "## Create a model\n", + "\n", + "Build the model locally as usual:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "from numpy import arange\n", + "from xarray import DataArray\n", + "\n", + "from linopy import Model\n", + "\n", + "N = 10\n", + "m = Model()\n", + "coords = [arange(N), arange(N)]\n", + "x = m.add_variables(coords=coords, name=\"x\")\n", + "y = m.add_variables(coords=coords, name=\"y\")\n", + "m.add_constraints(x - y >= DataArray(arange(N)))\n", + "m.add_constraints(x + y >= 0)\n", + "m.add_objective((2 * x + y).sum())\n", + "m" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "## SSH\n", + "\n", + "**What you need**\n", + "\n", + "- `uv pip install \"linopy[ssh]\"` locally (pulls in `paramiko`).\n", + "- A remote server with linopy and a solver installed (e.g. in a conda environment).\n", + "- SSH access to that machine (key-based auth recommended).\n", + "\n", + "Build an `SshSettings` and pass it as `remote=`. Use `setup_commands` to activate environments or export variables on the remote shell before the solve." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.remote import SshSettings\n", + "\n", + "ssh_settings = SshSettings(\n", + " hostname=\"your.host.de\",\n", + " username=\"username\",\n", + " # password=\"...\", # not needed when SSH keys are autodetected\n", + " setup_commands=[\"conda activate linopy-env\"],\n", + ")\n", + "\n", + "m.solve(\"gurobi\", remote=ssh_settings)\n", + "m.solution" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "## OETC\n", + "\n", + "**What you need**\n", + "\n", + "- `uv pip install \"linopy[oetc]\"` locally (pulls in `google-cloud-storage` and `requests`).\n", + "- An OETC account with valid credentials.\n", + "- The OETC authentication and orchestrator server URLs.\n", + "\n", + "Build an `OetcSettings`. Two construction styles:\n", + "\n", + "1. **Manually** — pass `email`, `password`, `name`, and the server URLs.\n", + "2. **`OetcSettings.from_env()`** — resolve everything from environment variables (`OETC_EMAIL`, `OETC_PASSWORD`, `OETC_NAME`, `OETC_AUTH_URL`, `OETC_ORCHESTRATOR_URL`). Recommended for CI/CD. Keyword arguments override the environment.\n", + "\n", + "linopy uploads the model to OETC, submits a compute job, polls until it finishes, and downloads the solution — all behind one call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.remote import OetcSettings\n", + "\n", + "# Option 1: pass credentials directly\n", + "oetc_settings = OetcSettings(\n", + " email=\"your-email@example.com\",\n", + " password=\"your-password\",\n", + " name=\"linopy-example-job\",\n", + " authentication_server_url=\"https://auth.oetcloud.com\",\n", + " orchestrator_server_url=\"https://orchestrator.oetcloud.com\",\n", + " cpu_cores=4,\n", + " disk_space_gb=20,\n", + ")\n", + "\n", + "# Option 2: load from environment (with optional overrides)\n", + "oetc_settings = OetcSettings.from_env(cpu_cores=4, disk_space_gb=20)\n", + "\n", + "m.solve(\"gurobi\", remote=oetc_settings, TimeLimit=600, MIPGap=0.01)\n", + "\n", + "print(f\"Status: {m.status}\")\n", + "print(f\"Objective: {m.objective.value:.4f}\")\n", + "m.solution" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "## Advanced: drive the transport directly\n", + "\n", + "For finer control — inspecting the round-tripped solved model, splitting submit from collect for async workflows — use the `Oetc` or `SSH` class directly. `Model.solve(remote=...)` runs the same path internally and then writes the result back onto the local model in place.\n", + "\n", + "### SSH" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.remote import SSH\n", + "\n", + "ssh = SSH(\n", + " settings=ssh_settings,\n", + " solver_name=\"gurobi\",\n", + " options={\"presolve\": \"on\"},\n", + ")\n", + "result = ssh.solve(m)\n", + "m.assign_result(result)" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "### OETC\n", + "\n", + "`Oetc` exposes the three steps `Model.solve(remote=...)` does internally:\n", + "\n", + "1. `upload(model)` — serialize and push the netcdf to OETC.\n", + "2. `submit()` — submit the compute job; returns the job uuid.\n", + "3. `collect(model)` — wait for completion, download, build the `Result`.\n", + "\n", + "Splitting them lets you fire off a job, do other work, and come back to collect later — useful for long-running jobs or async-style workflows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "from linopy.remote import Oetc\n", + "\n", + "oetc = Oetc(\n", + " settings=oetc_settings,\n", + " solver_name=\"gurobi\",\n", + " options={\"TimeLimit\": 600, \"MIPGap\": 0.01},\n", + ")\n", + "\n", + "oetc.upload(m)\n", + "job_uuid = oetc.submit()\n", + "print(f\"Submitted job {job_uuid} — do other work here ...\")\n", + "\n", + "# Later (or in another process holding `oetc`):\n", + "result = oetc.collect(m)\n", + "m.assign_result(result)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "nbsphinx": { + "execute": "never" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/solve-on-oetc.ipynb b/examples/solve-on-oetc.ipynb deleted file mode 100644 index 28e1c04d..00000000 --- a/examples/solve-on-oetc.ipynb +++ /dev/null @@ -1,431 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Solve on OETC (OET Cloud)\n", - "\n", - "This example demonstrates how to use linopy with OETC (OET Cloud) for cloud-based optimization solving. OETC is a cloud platform that provides scalable computing resources for optimization problems.\n", - "\n", - "## What you need to run this example:\n", - "\n", - "* A working installation of the required packages:\n", - " * `pip install google-cloud-storage requests`\n", - "* An OETC account with valid credentials (email and password)\n", - "* Access to OETC authentication and orchestrator servers\n", - "\n", - "## How OETC Cloud Solving Works\n", - "\n", - "The OETC integration follows this workflow:\n", - "\n", - "1. **Model Creation**: Define your optimization model locally using linopy\n", - "2. **Authentication**: Sign in to the OETC platform using your credentials\n", - "3. **File Upload**: Compress and upload your model to Google Cloud Storage\n", - "4. **Job Submission**: Submit a compute job to the OETC orchestrator\n", - "5. **Job Monitoring**: Wait for job completion with automatic status polling\n", - "6. **Solution Download**: Download and decompress the solved model\n", - "7. **Local Integration**: Load the solution back into your local model\n", - "\n", - "All of these steps are handled automatically by linopy's `OetcHandler`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> **Note:** This notebook requires Google Cloud credentials and access to the OETC platform. It is not executed during the documentation build, so no cell outputs are shown. To run it yourself, install the `linopy[oetc]` extra and configure your credentials." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create a Model\n", - "\n", - "First, let's create an optimization model that we want to solve on OETC:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from numpy import arange\n", - "from xarray import DataArray\n", - "\n", - "from linopy import Model\n", - "\n", - "# Create a medium-sized optimization problem\n", - "N = 50\n", - "m = Model()\n", - "\n", - "# Define decision variables with coordinates\n", - "coords = [arange(N), arange(N)]\n", - "x = m.add_variables(coords=coords, name=\"x\", lower=0)\n", - "y = m.add_variables(coords=coords, name=\"y\", lower=0)\n", - "\n", - "# Add constraints\n", - "m.add_constraints(x - y >= DataArray(arange(N)), name=\"constraint1\")\n", - "m.add_constraints(x + y >= DataArray(arange(N) * 0.5), name=\"constraint2\")\n", - "m.add_constraints(x <= DataArray(arange(N) + 10), name=\"upper_bounds\")\n", - "\n", - "# Set objective function\n", - "m.add_objective((2 * x + y).sum())\n", - "\n", - "print(\n", - " f\"Model created with {len(m.variables)} variable groups and {len(m.constraints)} constraint groups\"\n", - ")\n", - "m" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Configure OETC Settings\n", - "\n", - "There are two ways to configure OETC settings:\n", - "\n", - "1. **Manual construction** — build `OetcCredentials` and `OetcSettings` explicitly\n", - "2. **`OetcSettings.from_env()`** — resolve credentials and options from environment variables\n", - "\n", - "### Option 1: Manual Construction" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Configure your OETC credentials\n", - "# IMPORTANT: Never hardcode credentials in production code!\n", - "# Use environment variables or secure credential management\n", - "import os\n", - "\n", - "from linopy.remote.oetc import (\n", - " ComputeProvider,\n", - " OetcCredentials,\n", - " OetcHandler,\n", - " OetcSettings,\n", - ")\n", - "\n", - "credentials = OetcCredentials(\n", - " email=os.getenv(\"OETC_EMAIL\", \"your-email@example.com\"),\n", - " password=os.getenv(\"OETC_PASSWORD\", \"your-password\"),\n", - ")\n", - "\n", - "# Configure OETC settings\n", - "settings = OetcSettings(\n", - " credentials=credentials,\n", - " name=\"linopy-example-job\",\n", - " authentication_server_url=\"https://auth.oetcloud.com\", # Replace with actual URL\n", - " orchestrator_server_url=\"https://orchestrator.oetcloud.com\", # Replace with actual URL\n", - " compute_provider=ComputeProvider.GCP,\n", - " cpu_cores=4, # Number of CPU cores to allocate\n", - " disk_space_gb=20, # Disk space in GB\n", - " delete_worker_on_error=False, # Keep worker for debugging if job fails\n", - ")\n", - "\n", - "print(\"OETC settings configured successfully\")\n", - "print(f\"Solver: {settings.solver}\")\n", - "print(f\"CPU cores: {settings.cpu_cores}\")\n", - "print(f\"Disk space: {settings.disk_space_gb} GB\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Option 2: Create Settings from Environment Variables\n", - "\n", - "`OetcSettings.from_env()` reads configuration from environment variables,\n", - "with optional keyword overrides. This is the recommended approach for\n", - "CI/CD pipelines and production deployments.\n", - "\n", - "| Environment Variable | Required | Description |\n", - "|---|---|---|\n", - "| `OETC_EMAIL` | Yes | Account email |\n", - "| `OETC_PASSWORD` | Yes | Account password |\n", - "| `OETC_NAME` | Yes | Job name |\n", - "| `OETC_AUTH_URL` | Yes | Authentication server URL |\n", - "| `OETC_ORCHESTRATOR_URL` | Yes | Orchestrator server URL |\n", - "| `OETC_CPU_CORES` | No | CPU cores (default: 2) |\n", - "| `OETC_DISK_SPACE_GB` | No | Disk space in GB (default: 10) |\n", - "| `OETC_DELETE_WORKER_ON_ERROR` | No | Delete worker on error (default: false) |\n", - "\n", - "Keyword arguments take precedence over environment variables." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create settings from environment variables\n", - "# All required env vars must be set: OETC_EMAIL, OETC_PASSWORD,\n", - "# OETC_NAME, OETC_AUTH_URL, OETC_ORCHESTRATOR_URL\n", - "settings = OetcSettings.from_env()\n", - "\n", - "# Or override specific values via keyword arguments\n", - "settings = OetcSettings.from_env(\n", - " cpu_cores=8,\n", - " disk_space_gb=50,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Initialize OETC Handler\n", - "\n", - "The `OetcHandler` manages the entire cloud solving process:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize the OETC handler\n", - "# This will authenticate with OETC and fetch cloud provider credentials\n", - "oetc_handler = OetcHandler(settings)\n", - "\n", - "print(\"OETC handler initialized successfully\")\n", - "print(f\"Authentication token expires at: {oetc_handler.jwt.expires_at}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Solve the Model on OETC\n", - "\n", - "Now we can solve our model on the OETC cloud platform. The `OetcHandler` is passed to the model's `solve()` method:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Solve the model on OETC\n", - "# This will upload the model, submit a job, wait for completion, and download the solution\n", - "import time\n", - "\n", - "print(\"Starting cloud solving process...\")\n", - "start_time = time.time()\n", - "\n", - "try:\n", - " status, termination_condition = m.solve(remote=oetc_handler, solver_name=\"highs\")\n", - "\n", - " end_time = time.time()\n", - " total_time = end_time - start_time\n", - "\n", - " print(f\"\\nSolving completed in {total_time:.2f} seconds\")\n", - " print(f\"Status: {status}\")\n", - " print(f\"Termination condition: {termination_condition}\")\n", - " print(f\"Objective value: {m.objective.value:.4f}\")\n", - "\n", - "except Exception as e:\n", - " print(f\"Error during solving: {e}\")\n", - " raise" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Examine the Solution\n", - "\n", - "Let's examine the solution returned from OETC:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Display solution summary\n", - "print(f\"Model status: {m.status}\")\n", - "print(f\"Objective value: {m.objective.value}\")\n", - "print(f\"Number of variables: {m.solution.sizes}\")\n", - "\n", - "# Show a subset of the solution\n", - "print(\"\\nSample of solution values:\")\n", - "print(\"x values (first 5x5):\")\n", - "print(m.solution[\"x\"].isel(dim_0=slice(0, 5), dim_1=slice(0, 5)).values)\n", - "\n", - "print(\"\\ny values (first 5x5):\")\n", - "print(m.solution[\"y\"].isel(dim_0=slice(0, 5), dim_1=slice(0, 5)).values)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Advanced OETC Configuration\n", - "\n", - "### Solver Options\n", - "\n", - "Solver name and options can be configured at two levels:\n", - "\n", - "1. **Settings level** — defaults stored in `OetcSettings.solver` and `OetcSettings.solver_options`\n", - "2. **Call level** — passed via `m.solve(solver_name=..., **solver_options)`\n", - "\n", - "Call-level options **override** settings-level options. The two dicts are\n", - "merged (call-time takes precedence), and the original settings are never\n", - "mutated." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Settings-level defaults\n", - "advanced_settings = OetcSettings(\n", - " credentials=credentials,\n", - " name=\"advanced-linopy-job\",\n", - " authentication_server_url=\"https://auth.oetcloud.com\",\n", - " orchestrator_server_url=\"https://orchestrator.oetcloud.com\",\n", - " solver=\"gurobi\",\n", - " solver_options={\n", - " \"TimeLimit\": 600,\n", - " \"MIPGap\": 0.01,\n", - " },\n", - " cpu_cores=8,\n", - " disk_space_gb=50,\n", - ")\n", - "\n", - "advanced_handler = OetcHandler(advanced_settings)\n", - "\n", - "# Call-level overrides: solver_name and solver_options are forwarded\n", - "# to OETC and merged with the settings defaults.\n", - "# Here MIPGap from settings (0.01) is kept, TimeLimit is overridden to 300.\n", - "status, condition = m.solve(\n", - " remote=advanced_handler,\n", - " solver_name=\"gurobi\",\n", - " TimeLimit=300,\n", - " Threads=4,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Error Handling and Debugging\n", - "\n", - "When working with cloud solving, it's important to handle potential errors gracefully:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def solve_with_error_handling(model, oetc_handler, max_retries=3):\n", - " \"\"\"Solve model with error handling and retries\"\"\"\n", - "\n", - " for attempt in range(max_retries):\n", - " try:\n", - " print(f\"Solving attempt {attempt + 1}/{max_retries}...\")\n", - " status, termination = model.solve(remote=oetc_handler)\n", - "\n", - " if status == \"ok\":\n", - " print(\"Solving successful!\")\n", - " return status, termination\n", - " else:\n", - " print(f\"Solving returned status: {status}\")\n", - "\n", - " except Exception as e:\n", - " print(f\"Attempt {attempt + 1} failed: {e}\")\n", - "\n", - " if attempt < max_retries - 1:\n", - " print(\"Retrying in 30 seconds...\")\n", - " time.sleep(30)\n", - " else:\n", - " print(\"All attempts failed\")\n", - " raise\n", - "\n", - " return None, None\n", - "\n", - "\n", - "# Example usage (commented out to avoid actual execution)\n", - "# status, termination = solve_with_error_handling(m, oetc_handler)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Security Best Practices\n", - "\n", - "When using OETC in production:\n", - "\n", - "1. **Never hardcode credentials**: Use environment variables or secure credential stores\n", - "2. **Use token expiration**: The OETC handler automatically manages token expiration\n", - "3. **Validate inputs**: Ensure your model data doesn't contain sensitive information\n", - "4. **Monitor costs**: Cloud computing resources have associated costs\n", - "5. **Clean up resources**: Set `delete_worker_on_error=True` for automatic cleanup\n", - "\n", - "## Comparison with SSH Remote Solving\n", - "\n", - "| Feature | OETC Cloud | SSH Remote |\n", - "|---------|------------|------------|\n", - "| Setup | Account registration | Server access required |\n", - "| Scalability | Auto-scaling | Fixed server resources |\n", - "| Maintenance | Managed service | Self-managed |\n", - "| Cost | Pay-per-use | Infrastructure costs |\n", - "| Security | Enterprise-grade | Self-managed |\n", - "| Solver Licenses | Included | User-provided |\n", - "\n", - "Choose OETC for:\n", - "- Large-scale problems requiring significant compute resources\n", - "- Temporary or intermittent optimization needs\n", - "- Teams without dedicated infrastructure\n", - "- Access to premium solvers without license management\n", - "\n", - "Choose SSH remote for:\n", - "- Existing infrastructure with optimization solvers\n", - "- Strict data governance requirements\n", - "- Consistent, long-running optimization workloads\n", - "- Full control over the solving environment" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - }, - "nbsphinx": { - "execute": "never" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/solve-on-remote.ipynb b/examples/solve-on-remote.ipynb deleted file mode 100644 index 73e6346b..00000000 --- a/examples/solve-on-remote.ipynb +++ /dev/null @@ -1,655 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4db583af", - "metadata": {}, - "source": [ - "# Remote Solving with SSH\n", - "\n", - "This example demonstrates how linopy can solve optimization models on remote machines using SSH connections. This is one of two remote solving options available in linopy:\n", - "\n", - "1. **SSH Remote Solving** (this example) - Connect to your own servers via SSH\n", - "2. **OETC Cloud Solving** - Use cloud-based optimization services (see [OETC notebook](solve-on-oetc.ipynb))\n", - "\n", - "## SSH Remote Solving\n", - "\n", - "SSH remote solving is ideal when you have:\n", - "\n", - "* Access to dedicated servers with optimization solvers installed\n", - "* Full control over the computing environment\n", - "* Existing infrastructure for optimization workloads\n", - "\n", - "## What you need for SSH remote solving\n", - "\n", - "* The `remote` extra installed on your local machine (`uv pip install \"linopy[remote]\"`), which pulls in `paramiko`\n", - "* A remote server with a working installation of linopy (e.g., in a conda environment)\n", - "* SSH access to that machine\n", - "\n", - "## How SSH Remote Solving Works\n", - "\n", - "The workflow consists of the following steps, most of which linopy handles automatically:\n", - "\n", - "1. Define a model on the local machine\n", - "2. Save the model on the remote machine via SSH\n", - "3. Load, solve and write out the model on the remote machine\n", - "4. Copy the solved model back to the local machine\n", - "5. Load the solved model on the local machine\n", - "\n", - "The model initialization happens locally, while the actual solving happens remotely.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> **Note:** This notebook requires SSH access to a remote server with a solver installed. It is not executed during the documentation build, so no cell outputs are shown. To run it yourself, configure SSH access and install a solver on the remote machine." - ] - }, - { - "cell_type": "markdown", - "id": "together-ocean", - "metadata": {}, - "source": [ - "## Create a model\n", - "\n", - "First we are going to build the optimization model we want to solve in our local process." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "dramatic-cannon", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Linopy LP model\n", - "===============\n", - "\n", - "Variables:\n", - "----------\n", - " * x (dim_0, dim_1)\n", - " * y (dim_0, dim_1)\n", - "\n", - "Constraints:\n", - "------------\n", - " * con0 (dim_0, dim_1)\n", - " * con1 (dim_0, dim_1)\n", - "\n", - "Status:\n", - "-------\n", - "initialized" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from numpy import arange\n", - "from xarray import DataArray\n", - "\n", - "from linopy import Model\n", - "\n", - "N = 10\n", - "m = Model()\n", - "coords = [arange(N), arange(N)]\n", - "x = m.add_variables(coords=coords, name=\"x\")\n", - "y = m.add_variables(coords=coords, name=\"y\")\n", - "m.add_constraints(x - y >= DataArray(arange(N)))\n", - "m.add_constraints(x + y >= 0)\n", - "m.add_objective((2 * x + y).sum())\n", - "m" - ] - }, - { - "cell_type": "markdown", - "id": "0f9e9b09", - "metadata": {}, - "source": [ - "## Initialize SSH connection\n", - "\n", - "Now we have to set up the SSH connection. The SSH connection is handled by the `RemoteHandler` class in of the `linopy.remote` module. This is strongly relying on the `paramiko` package. When initializing, you have two options:\n", - "\n", - "1. Pass the standard arguments `host`, `username`. If the SSH keys are stored in a default location, the keys are autodetected and the `RemoteHandler` does not require the `password` argument. Otherwise you also have to pass the password.\n", - "2. Pass a working `paramiko.SSHClient` as `client`. This enables you to set up the SSH connection by others means supported by `paramiko`. \n", - "\n", - "In the following we use the first option." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "protecting-power", - "metadata": {}, - "outputs": [], - "source": [ - "from linopy import RemoteHandler\n", - "\n", - "host = \"your.host.de\"\n", - "username = \"username\"\n", - "\n", - "handler = RemoteHandler(host, username=username)" - ] - }, - { - "cell_type": "markdown", - "id": "featured-maria", - "metadata": {}, - "source": [ - "## Optionally: Activate a conda environment on the remote \n", - "\n", - "The `RemoteHandler` keeps an interactive shell in the background. You can execute any code in order to prepare the solving process (install linopy, activate an environment). \n", - "\n", - "Assuming you have a conda environment `linopy-env` that contains the `linopy` package with dependencies, you can run " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "virtual-anxiety", - "metadata": {}, - "outputs": [], - "source": [ - "handler.execute(\"conda activate linopy-env\")" - ] - }, - { - "cell_type": "markdown", - "id": "sonic-rebate", - "metadata": {}, - "source": [ - "## Solve the model on remote\n", - "\n", - "Now the only thing you have to do is to pass the `RemoteHandler` as an argument to the `solve` function. Other keyword arguments like `solver_name` and solver options are propagated to the remote machine. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "ongoing-desktop", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Set parameter Username\n", - "Academic license - for non-commercial use only - expires 2023-02-06\n", - "Read LP format model from file /tmp/linopy-problem-uh4gvjyp.lp\n", - "Reading time = 0.00 seconds\n", - "obj: 200 rows, 200 columns, 400 nonzeros\n", - "Gurobi Optimizer version 9.5.1 build v9.5.1rc2 (linux64)\n", - "Thread count: 12 physical cores, 24 logical processors, using up to 24 threads\n", - "Optimize a model with 200 rows, 200 columns and 400 nonzeros\n", - "Model fingerprint: 0xf2bcac49\n", - "Coefficient statistics:\n", - "Matrix range [1e+00, 1e+00]\n", - "Objective range [1e+00, 2e+00]\n", - "Bounds range [0e+00, 0e+00]\n", - "RHS range [1e+00, 9e+00]\n", - "Presolve removed 200 rows and 200 columns\n", - "Presolve time: 0.00s\n", - "Presolve: All rows and columns removed\n", - "Iteration Objective Primal Inf. Dual Inf. Time\n", - "0 2.2500000e+02 0.000000e+00 0.000000e+00 0s\n", - "\n", - "Solved in 0 iterations and 0.00 seconds (0.00 work units)\n", - "Optimal objective 2.250000000e+02\n" - ] - }, - { - "data": { - "text/plain": [ - "('ok', '')" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "m.solve(remote=handler)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "sustained-portrait", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
<xarray.Dataset>\n",
-       "Dimensions:  (dim_0: 10, dim_1: 10)\n",
-       "Coordinates:\n",
-       "  * dim_0    (dim_0) int64 0 1 2 3 4 5 6 7 8 9\n",
-       "  * dim_1    (dim_1) int64 0 1 2 3 4 5 6 7 8 9\n",
-       "Data variables:\n",
-       "    x        (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... 4.5 4.5 4.5 4.5 4.5\n",
-       "    y        (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... -4.5 -4.5 -4.5 -4.5
" - ], - "text/plain": [ - "\n", - "Dimensions: (dim_0: 10, dim_1: 10)\n", - "Coordinates:\n", - " * dim_0 (dim_0) int64 0 1 2 3 4 5 6 7 8 9\n", - " * dim_1 (dim_1) int64 0 1 2 3 4 5 6 7 8 9\n", - "Data variables:\n", - " x (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... 4.5 4.5 4.5 4.5 4.5\n", - " y (dim_0, dim_1) float64 0.0 0.0 0.0 0.0 0.0 ... -4.5 -4.5 -4.5 -4.5" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "m.solution" - ] - } - ], - "metadata": { - "@webio": { - "lastCommId": null, - "lastKernelId": null - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - }, - "nbsphinx": { - "execute": "never" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/linopy/model.py b/linopy/model.py index 48a8200b..d88a7534 100644 --- a/linopy/model.py +++ b/linopy/model.py @@ -79,11 +79,13 @@ add_piecewise_formulation, ) from linopy.remote import RemoteHandler +from linopy.remote.ssh import SshSettings try: - from linopy.remote import OetcHandler + from linopy.remote import OetcHandler, OetcSettings except ImportError: OetcHandler = None # type: ignore + OetcSettings = None # type: ignore from linopy.solver_capabilities import solver_supports from linopy.solvers import ( IO_APIS, @@ -94,6 +96,7 @@ SOSReformulationResult, reformulate_sos_constraints, sos_reformulation_context, + suppress_serialization_warning, undo_sos_reformulation, ) from linopy.types import ( @@ -111,6 +114,14 @@ logger = logging.getLogger(__name__) +# Types accepted as ``remote=`` for the standalone-class dispatch in +# :meth:`Model.solve` (as opposed to the legacy ``OetcHandler`` / +# ``RemoteHandler`` deprecation path). The OETC entry is conditional on +# the optional google-cloud / requests deps being available. +_REMOTE_SETTINGS_TYPES: tuple[type, ...] = (SshSettings,) +if OetcSettings is not None: + _REMOTE_SETTINGS_TYPES = (*_REMOTE_SETTINGS_TYPES, OetcSettings) + def _coords_to_dict( coords: Sequence[Sequence | pd.Index | DataArray] | Mapping, @@ -196,6 +207,7 @@ class Model: """ _solver: solvers.Solver | None + _remote: Any _variables: Variables _constraints: Constraints _objective: Objective @@ -243,6 +255,7 @@ class Model: "_relaxed_registry", "_piecewise_formulations", "_solver", + "_remote", "_sos_reformulation_state", "__weakref__", ) @@ -314,6 +327,7 @@ def __init__( gettempdir() if solver_dir is None else solver_dir ) self._solver: solvers.Solver | None = None + self._remote: Any = None self._sos_reformulation_state: SOSReformulationResult | None = None @property @@ -326,6 +340,24 @@ def solver(self, value: solvers.Solver | None) -> None: self._solver.close() self._solver = value + @property + def remote(self) -> Any: + """ + Standalone remote-handler instance from the most recent solve, or ``None``. + + Set by :meth:`solve` when called with ``remote=``; lets + callers introspect handler state after the solve (e.g. + ``model.remote._job_uuid`` on OETC). ``None`` for local solves + and after a legacy ``remote=OetcHandler/RemoteHandler`` solve + (those are routed through the same path but the legacy handlers + aren't designed for post-solve inspection). + """ + return self._remote + + @remote.setter + def remote(self, value: Any) -> None: + self._remote = value + @property def solver_model(self) -> Any: return self.solver.solver_model if self.solver is not None else None @@ -1594,7 +1626,7 @@ def solve( sanitize_zeros: bool = True, sanitize_infinities: bool = True, slice_size: int = 2_000_000, - remote: RemoteHandler | OetcHandler | None = None, + remote: RemoteHandler | OetcHandler | OetcSettings | SshSettings | None = None, progress: bool | None = None, mock_solve: bool = False, reformulate_sos: bool | Literal["auto"] = False, @@ -1699,50 +1731,37 @@ def solve( f"Keyword argument `io_api` has to be one of {IO_APIS} or None" ) - if remote is not None: - # The remote branch short-circuits before reaching Solver.solve(), - # which is where the empty-objective check normally fires. Replicate - # it here. This duplication becomes obsolete once OETC is folded - # into the Solver pipeline (see PyPSA/linopy#683). - if self.objective.expression.empty: - raise ValueError( - "No objective has been set on the model. Use " - "`m.add_objective(...)` first (e.g. `m.add_objective(0 * x)` " - "for a pure feasibility problem)." - ) - if isinstance(remote, OetcHandler): - solved = remote.solve_on_oetc( - self, - solver_name=solver_name, - reformulate_sos=reformulate_sos, - **solver_options, - ) - else: - solved = remote.solve_on_remote( - self, - solver_name=solver_name, - io_api=io_api, - problem_fn=problem_fn, - solution_fn=solution_fn, - log_fn=log_fn, - basis_fn=basis_fn, - warmstart_fn=warmstart_fn, - keep_files=keep_files, - sanitize_zeros=sanitize_zeros, - reformulate_sos=reformulate_sos, - **solver_options, - ) + # New standalone Oetc / SSH remote handlers are selected by passing + # their settings dataclass via ``remote=``. ``solver_name`` and + # ``**solver_options`` describe the *inner* solver to run on the + # worker. + if isinstance(remote, _REMOTE_SETTINGS_TYPES): + return self._solve_with_remote_settings( + remote, + inner_solver=solver_name, + solver_options=solver_options, + reformulate_sos=reformulate_sos, + ) - if solved.objective.value is not None: - self.objective.set_value(float(solved.objective.value)) - self.status = solved.status - self.termination_condition = solved.termination_condition - for k, v in self.variables.items(): - v.solution = solved.variables[k].solution - for k, c in self.constraints.items(): - if "dual" in solved.constraints[k]: - c.dual = solved.constraints[k].dual - return self.status, self.termination_condition + if remote is not None: + # Back-compat shim: the legacy ``remote=OetcHandler/RemoteHandler`` + # shape pre-dates the standalone Oetc/SSH classes. Route to the + # new entrypoint and warn. Slated for removal once one release of + # overlap has shipped. + return self._solve_via_legacy_remote( + remote, + solver_name=solver_name, + io_api=io_api, + problem_fn=problem_fn, + solution_fn=solution_fn, + log_fn=log_fn, + basis_fn=basis_fn, + warmstart_fn=warmstart_fn, + keep_files=keep_files, + sanitize_zeros=sanitize_zeros, + reformulate_sos=reformulate_sos, + solver_options=solver_options, + ) if len(available_solvers) == 0: raise RuntimeError("No solver installed.") @@ -1827,6 +1846,173 @@ def solve( return self.assign_result(result) + def _solve_with_remote_settings( + self, + settings: Any, + *, + inner_solver: str | None, + solver_options: dict[str, Any], + reformulate_sos: bool | Literal["auto"], + ) -> tuple[str, str]: + """ + Dispatch a remote solve from an ``OetcSettings`` / ``SshSettings`` instance. + + The new standalone remote handlers (``Oetc``, ``SSH`` in + :mod:`linopy.remote`) are *not* :class:`linopy.solvers.Solver` + subclasses — they're a parallel concept. The instance is attached + to :attr:`Model.remote` after the call so callers can introspect + e.g. the OETC job uuid. + """ + effective_inner: str | None + effective_options: dict[str, Any] + if OetcSettings is not None and isinstance(settings, OetcSettings): + from linopy.remote.oetc import Oetc + + remote_cls: Any = Oetc + # ``OetcSettings`` carries defaults for solver/solver_options + # (preserves the legacy ``OetcHandler(settings).solve_on_oetc`` + # config style). Outer ``Model.solve(solver_name, **opts)`` + # wins when given. + effective_inner = inner_solver or settings.solver + effective_options = {**settings.solver_options, **solver_options} + elif isinstance(settings, SshSettings): + from linopy.remote.ssh import SSH + + remote_cls = SSH + effective_inner = inner_solver + effective_options = solver_options + else: + raise TypeError( # pragma: no cover — checked by _REMOTE_SETTINGS_TYPES + f"Unknown remote settings type: {type(settings).__name__}" + ) + + if not effective_inner: + raise ValueError( + f"`m.solve(remote=<{type(settings).__name__}>)` requires " + "an explicit `solver_name=` for the solver to run " + "on the worker." + ) + + if self.objective.expression.empty: + raise ValueError( + "No objective has been set on the model. Use " + "`m.add_objective(...)` first (e.g. `m.add_objective(0 * x)` " + "for a pure feasibility problem)." + ) + + # Apply SOS reformulation before the remote handler serializes the + # model; the worker just solves a plain MILP, the lifecycle stays + # on this Model. ``sos_reformulation_context`` handles the + # apply/undo bracket, ``suppress_serialization_warning`` silences + # the ``to_netcdf`` UserWarning that fires when serializing in + # reformulated form (intentional here). + with sos_reformulation_context( + self, effective_inner, reformulate_sos + ) as applied: + with suppress_serialization_warning(active=applied): + remote_instance = remote_cls( + settings=settings, + solver_name=effective_inner, + options=effective_options, + ) + self.remote = remote_instance + self.solver = None # remote-solve clears any prior local solver + result = remote_instance.solve(self) + return self.assign_result(result) + + def _solve_via_legacy_remote( + self, + remote: Any, + *, + solver_name: str | None, + io_api: str | None, + problem_fn: str | Path | None, + solution_fn: str | Path | None, + log_fn: str | Path | None, + basis_fn: str | Path | None, + warmstart_fn: str | Path | None, + keep_files: bool, + sanitize_zeros: bool, + reformulate_sos: bool | Literal["auto"], + solver_options: dict[str, Any], + ) -> tuple[str, str]: + """ + Back-compat path for ``Model.solve(remote=)``. + + Calls ``handler.solve_on_oetc(...)`` / ``handler.solve_on_remote(...)`` + as before — preserves the behavior tests on master are asserting + against — and emits a :class:`DeprecationWarning` pointing users at + the new ``remote=`` shape. + """ + if OetcHandler is not None and isinstance(remote, OetcHandler): + warnings.warn( + "Passing an OetcHandler via `remote=` is deprecated; pass " + "the OetcSettings directly: " + "`m.solve(remote=OetcSettings(...))`. The " + "`remote=OetcHandler/RemoteHandler` shape will be removed " + "in a future release.", + DeprecationWarning, + stacklevel=3, + ) + elif isinstance(remote, RemoteHandler): + warnings.warn( + "Passing a RemoteHandler via `remote=` is deprecated; pass " + "an SshSettings via `remote=` with a `solver_name=` for " + "the solver (`m.solve(solver_name, remote=SshSettings" + "(...))`). The `remote=OetcHandler/RemoteHandler` shape " + "will be removed in a future release.", + DeprecationWarning, + stacklevel=3, + ) + else: + raise TypeError( + f"`remote` must be an OetcHandler, RemoteHandler, " + f"OetcSettings, or SshSettings, got {type(remote).__name__}" + ) + + # The remote handlers short-circuit before reaching Solver.solve(), + # which is where the empty-objective check normally fires. Replicate + # it here. + if self.objective.expression.empty: + raise ValueError( + "No objective has been set on the model. Use " + "`m.add_objective(...)` first (e.g. `m.add_objective(0 * x)` " + "for a pure feasibility problem)." + ) + if OetcHandler is not None and isinstance(remote, OetcHandler): + solved = remote.solve_on_oetc( + self, + solver_name=solver_name, + reformulate_sos=reformulate_sos, + **solver_options, + ) + else: + solved = remote.solve_on_remote( + self, + solver_name=solver_name, + io_api=io_api, + problem_fn=problem_fn, + solution_fn=solution_fn, + log_fn=log_fn, + basis_fn=basis_fn, + warmstart_fn=warmstart_fn, + keep_files=keep_files, + sanitize_zeros=sanitize_zeros, + reformulate_sos=reformulate_sos, + **solver_options, + ) + + if solved.objective.value is not None: + self.objective.set_value(float(solved.objective.value)) + self.status = solved.status + self.termination_condition = solved.termination_condition + for k, v in self.variables.items(): + v.solution = solved.variables[k].solution + for k, c in self.constraints.items(): + if "dual" in solved.constraints[k]: + c.dual = solved.constraints[k].dual + return self.status, self.termination_condition + def assign_result( self, result: Result, diff --git a/linopy/remote/__init__.py b/linopy/remote/__init__.py index d3d5e162..c8642ec2 100644 --- a/linopy/remote/__init__.py +++ b/linopy/remote/__init__.py @@ -8,16 +8,19 @@ - OetcHandler: Cloud-based execution via OET Cloud service """ -from linopy.remote.ssh import RemoteHandler +from linopy.remote.ssh import SSH, RemoteHandler, SshSettings try: - from linopy.remote.oetc import OetcCredentials, OetcHandler, OetcSettings + from linopy.remote.oetc import Oetc, OetcCredentials, OetcHandler, OetcSettings except ImportError: pass __all__ = [ "RemoteHandler", + "SSH", + "SshSettings", "OetcHandler", + "Oetc", "OetcSettings", "OetcCredentials", ] diff --git a/linopy/remote/_common.py b/linopy/remote/_common.py new file mode 100644 index 00000000..71719680 --- /dev/null +++ b/linopy/remote/_common.py @@ -0,0 +1,85 @@ +""" +Shared helpers for the standalone remote-handler classes (``Oetc``, ``SSH``). + +These handlers do not inherit from :class:`linopy.solvers.Solver` — they're +a parallel concept. The helpers here cover the two pieces of plumbing +both handlers need: validating the inner-solver string locally, and +mapping a round-tripped solved :class:`~linopy.model.Model` back onto +the source model's label space. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np + +from linopy.constants import Solution + +if TYPE_CHECKING: + from linopy.model import Model + + +def _validate_inner_solver(inner_solver_name: str, model: Model) -> None: + """ + Check that the inner-solver string is locally known and + that the inner solver's feature set covers the model. + + Local installation is *not* required — feature flags are class-level + metadata. We only need the class to introspect ``supports(...)``. + Unknown solver names raise so typos fail fast instead of incurring a + round-trip to the worker. + """ + # Imported here to avoid a circular import at module load. + from linopy.solvers import SolverFeature, SolverName, _solver_class_for + + cls = _solver_class_for(inner_solver_name) + if cls is None: + valid = ", ".join(sorted(n.value for n in SolverName)) + raise ValueError( + f"Unknown solver name {inner_solver_name!r}. Pick one of: {valid}." + ) + if model.is_quadratic and not cls.supports(SolverFeature.QUADRATIC_OBJECTIVE): + raise ValueError( + f"Solver {inner_solver_name!r} does not support quadratic problems." + ) + if model.variables.semi_continuous and not cls.supports( + SolverFeature.SEMI_CONTINUOUS_VARIABLES + ): + raise ValueError( + f"Solver {inner_solver_name!r} does not support semi-continuous " + "variables. Use a solver that supports them (gurobi, cplex, highs)." + ) + if model.variables.sos and not cls.supports(SolverFeature.SOS_CONSTRAINTS): + raise ValueError( + f"Solver {inner_solver_name!r} does not support SOS constraints. " + "Reformulate first via `Model.solve(reformulate_sos=True)` or " + "`model.apply_sos_reformulation()`, or pick a solver that supports SOS." + ) + + +def _scatter_solution_from_solved_model( + local_model: Model, solved: Model, n_vars: int, n_cons: int +) -> Solution: + """ + Build a label-indexed :class:`~linopy.constants.Solution` from a + round-tripped solved model. + + The labels on ``solved`` match ``local_model`` because both sides + serialize/load with the same linopy version; we use the local labels + as the index. Missing slots stay ``NaN``; constraints without + ``dual`` are skipped. + """ + primal = np.full(n_vars, np.nan, dtype=float) + dual = np.full(n_cons, np.nan, dtype=float) + for name, var in local_model.variables.items(): + sol = solved.variables[name].solution + primal[var.labels.values.ravel()] = sol.values.ravel() + for name, con in local_model.constraints.items(): + if "dual" not in solved.constraints[name]: + continue + dual[con.labels.values.ravel()] = solved.constraints[name].dual.values.ravel() + + objective_value = solved.objective.value + objective = float(objective_value) if objective_value is not None else float("nan") + return Solution(primal=primal, dual=dual, objective=objective) diff --git a/linopy/remote/oetc.py b/linopy/remote/oetc.py index beef5873..eeb31d19 100644 --- a/linopy/remote/oetc.py +++ b/linopy/remote/oetc.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import contextlib import gzip import json import logging @@ -12,6 +13,8 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Literal +from linopy.constants import Result, SolverReport, Status + if TYPE_CHECKING: from linopy.model import Model @@ -25,6 +28,8 @@ except ImportError: _oetc_deps_available = False +import warnings + import linopy from linopy.sos_reformulation import ( sos_reformulation_context, @@ -40,16 +45,45 @@ class ComputeProvider(str, Enum): @dataclass class OetcCredentials: + """ + .. deprecated:: + Pass ``email`` and ``password`` directly to :class:`OetcSettings` + instead of wrapping them in ``OetcCredentials``. This class will be + removed in a future release. + """ + email: str password: str + def __post_init__(self) -> None: + warnings.warn( + "`OetcCredentials` is deprecated; pass `email=` and `password=` " + "directly to `OetcSettings`. `OetcCredentials` will be removed " + "in a future release.", + DeprecationWarning, + stacklevel=2, + ) + @dataclass class OetcSettings: - credentials: OetcCredentials + """ + Config for the OET Cloud (OETC) remote service. + + Carries the auth/orchestrator endpoints, the worker resource sizing, + and **defaults** for the solver and its options. The defaults + can be overridden per call: + + >>> m.solve("gurobi", remote=OetcSettings(...), Method=2) # doctest: +SKIP + >>> m.solve(remote=OetcSettings(..., solver="gurobi")) # doctest: +SKIP + """ + name: str authentication_server_url: str orchestrator_server_url: str + email: str | None = None + password: str | None = None + credentials: OetcCredentials | None = None compute_provider: ComputeProvider = ComputeProvider.GCP solver: str = "highs" solver_options: dict[str, Any] = field(default_factory=dict) @@ -57,6 +91,19 @@ class OetcSettings: disk_space_gb: int = 10 delete_worker_on_error: bool = False + def __post_init__(self) -> None: + if self.credentials is not None: + # `credentials=` warns from its own __post_init__; carry its + # values over unless `email` / `password` were also explicitly + # given (in which case the call site wins). + if self.email is None: + self.email = self.credentials.email + if self.password is None: + self.password = self.credentials.password + self.credentials = None + if not self.email or not self.password: + raise ValueError("`OetcSettings` requires `email` and `password`.") + @classmethod def from_env( cls, @@ -100,9 +147,8 @@ def from_env( ) kwargs: dict[str, Any] = { - "credentials": OetcCredentials( - email=resolved["email"], password=resolved["password"] - ), + "email": resolved["email"], + "password": resolved["password"], "name": resolved["name"], "authentication_server_url": resolved["authentication_server_url"], "orchestrator_server_url": resolved["orchestrator_server_url"], @@ -185,12 +231,30 @@ class JobResult: class OetcHandler: - def __init__(self, settings: OetcSettings) -> None: + """ + .. deprecated:: + Use :class:`~linopy.remote.Oetc` or :meth:`Model.solve(remote=OetcSettings(...)) + ` instead. This class will be removed in a + future release. The new :class:`Oetc` class owns the public lifecycle + (``upload`` / ``submit`` / ``collect`` / ``solve``); ``OetcHandler`` + remains only for back-compat with code that holds a long-lived + handler instance. + """ + + def __init__(self, settings: OetcSettings, *, _internal: bool = False) -> None: if not _oetc_deps_available: raise ImportError( "The 'google-cloud-storage' and 'requests' packages are required " "for OetcHandler. Install them with: pip install linopy[oetc]" ) + if not _internal: + warnings.warn( + "`OetcHandler` is deprecated; use `Oetc(settings, solver_name, " + "options)` from `linopy.remote` or `Model.solve(remote=OetcSettings" + "(...))`. `OetcHandler` will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) self.settings = settings self.jwt = self.__sign_in() self.cloud_provider_credentials = self.__get_cloud_provider_credentials() @@ -208,8 +272,8 @@ def __sign_in(self) -> AuthenticationResult: try: logger.info("OETC - Signing in...") payload = { - "email": self.settings.credentials.email, - "password": self.settings.credentials.password, + "email": self.settings.email, + "password": self.settings.password, } response = requests.post( @@ -645,11 +709,17 @@ def solve_on_oetc( """ Solve a linopy model on the OET Cloud compute app. + .. deprecated:: + Use :class:`Oetc` or + :meth:`Model.solve(remote=OetcSettings(...)) `. + Parameters ---------- model : linopy.model.Model solver_name : str, optional Override the solver from settings. + reformulate_sos : bool | "auto", optional + See :meth:`linopy.model.Model.solve`. **solver_options Override/extend solver_options from settings. @@ -657,55 +727,36 @@ def solve_on_oetc( ------- linopy.model.Model Solved model. - - Raises - ------ - Exception: If solving fails at any stage """ + # Delegates to ``Oetc.solve`` so the upload→submit→poll→download + # orchestration lives in one place. This handler is reused as the + # underlying transport so existing auth/credentials are not refetched. + effective_solver = solver_name or self.settings.solver + merged_solver_options = {**self.settings.solver_options, **solver_options} + + oetc = Oetc( + settings=self.settings, + solver_name=effective_solver, + options=merged_solver_options, + ) + oetc._handler = self try: - effective_solver = solver_name or self.settings.solver - merged_solver_options = {**self.settings.solver_options, **solver_options} - with sos_reformulation_context( model, effective_solver, reformulate_sos ) as applied: - with tempfile.NamedTemporaryFile(prefix="linopy-", suffix=".nc") as fn: - fn.file.close() - with suppress_serialization_warning(active=applied): - model.to_netcdf(fn.name) - input_file_name = self._upload_file_to_gcp(fn.name) - - job_uuid = self._submit_job_to_compute_service( - input_file_name, effective_solver, merged_solver_options - ) - job_result = self.wait_and_get_job_data(job_uuid) - - if not job_result.output_files: - raise Exception("No output files found in completed job") - - output_file_name = job_result.output_files[0] - if isinstance(output_file_name, dict) and "name" in output_file_name: - output_file_name = output_file_name["name"] - - solution_file_path = self._download_file_from_gcp(output_file_name) - - solved_model = linopy.read_netcdf(solution_file_path) - - os.remove(solution_file_path) - - logger.info( - f"OETC - Model solved successfully. Status: {solved_model.status}" - ) - if solved_model.objective.value is not None: - logger.info( - f"OETC - Objective value: {solved_model.objective.value:.2e}" - ) - - return solved_model - + with suppress_serialization_warning(active=applied): + oetc.upload(model) + oetc.submit() + oetc.collect(model) except Exception as e: raise Exception(f"Error solving model on OETC: {e}") from e + solved_model = oetc._solved_model + logger.info(f"OETC - Model solved successfully. Status: {solved_model.status}") + if solved_model.objective.value is not None: + logger.info(f"OETC - Objective value: {solved_model.objective.value:.2e}") + return solved_model + def _gzip_compress(self, source_path: str) -> str: """ Compress a file using gzip compression. @@ -786,3 +837,114 @@ def _upload_file_to_gcp(self, file_path: str) -> str: except Exception as e: raise Exception(f"Failed to upload file to GCP: {e}") + + +@dataclass +class Oetc: + """ + Remote handler that solves a linopy model on the OET Cloud (OETC) service. + + This is a standalone class — *not* a :class:`linopy.solvers.Solver` + subclass. It ships a netcdf to a cloud worker which runs the inner + solver (``solver_name``) and returns a solved netcdf. The lifecycle + splits into ``upload`` / ``submit`` / ``collect`` so future async work + can drive the seam without changing callers. + + Parameters + ---------- + settings : OetcSettings + Auth + orchestrator config (where to talk to). + solver_name : str + Solver to run on the worker (e.g. ``"gurobi"``, ``"highs"``). + options : dict, optional + Solver options passed through to the solver. + + Notes + ----- + Construction is cheap; network I/O happens at :meth:`upload` / + :meth:`submit` / :meth:`collect`. :meth:`solve` runs all three + synchronously. + """ + + settings: OetcSettings + solver_name: str + options: dict[str, Any] = field(default_factory=dict) + + _handler: OetcHandler | None = field(init=False, default=None, repr=False) + _input_file_name: str | None = field(init=False, default=None, repr=False) + _job_uuid: str | None = field(init=False, default=None, repr=False) + _solved_model: Any = field(init=False, default=None, repr=False) + _n_vars: int = field(init=False, default=0, repr=False) + _n_cons: int = field(init=False, default=0, repr=False) + + @classmethod + def is_available(cls) -> bool: + """Return True iff the OETC network deps are importable.""" + return _oetc_deps_available + + def upload(self, model: Model) -> None: + """Serialize the model to netcdf and upload it to the cloud bucket.""" + if self._handler is None: + self._handler = OetcHandler(self.settings, _internal=True) + self._n_vars = model._xCounter + self._n_cons = model._cCounter + + with tempfile.NamedTemporaryFile(prefix="linopy-", suffix=".nc") as fn: + fn.file.close() + model.to_netcdf(fn.name) + self._input_file_name = self._handler._upload_file_to_gcp(fn.name) + + def submit(self) -> str: + """Submit the prepared job to the orchestrator; return the job uuid.""" + if self._handler is None or self._input_file_name is None: + raise RuntimeError("Call `upload(model)` before `submit()`.") + self._job_uuid = self._handler._submit_job_to_compute_service( + self._input_file_name, self.solver_name, dict(self.options) + ) + return self._job_uuid + + def collect(self, model: Model) -> Result: + """Poll, download, parse, and return a label-indexed Result.""" + from linopy.remote._common import _scatter_solution_from_solved_model + + if self._handler is None or self._job_uuid is None: + raise RuntimeError( + "Call `upload(model)` and `submit()` before `collect()`." + ) + + job_result = self._handler.wait_and_get_job_data(self._job_uuid) + if not job_result.output_files: + raise Exception("No output files found in completed job") + output_file_name = job_result.output_files[0] + if isinstance(output_file_name, dict) and "name" in output_file_name: + output_file_name = output_file_name["name"] + + solution_file_path = self._handler._download_file_from_gcp(output_file_name) + try: + solved = linopy.read_netcdf(solution_file_path) + finally: + with contextlib.suppress(OSError): + os.remove(solution_file_path) + + self._solved_model = solved + + status = Status.from_termination_condition(solved.termination_condition) + solution = _scatter_solution_from_solved_model( + model, solved, self._n_vars, self._n_cons + ) + report = SolverReport(runtime=job_result.duration_in_seconds) + return Result( + status=status, + solution=solution, + solver_name=self.solver_name, + report=report, + ) + + def solve(self, model: Model) -> Result: + """Run the full upload → submit → collect pipeline synchronously.""" + from linopy.remote._common import _validate_inner_solver + + _validate_inner_solver(self.solver_name, model) + self.upload(model) + self.submit() + return self.collect(model) diff --git a/linopy/remote/ssh.py b/linopy/remote/ssh.py index 7c0a0644..4638b5ab 100644 --- a/linopy/remote/ssh.py +++ b/linopy/remote/ssh.py @@ -8,10 +8,12 @@ import logging import os import tempfile +import warnings from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Literal, Union +from linopy.constants import Result from linopy.io import read_netcdf from linopy.sos_reformulation import ( sos_reformulation_context, @@ -37,11 +39,42 @@ """ +@dataclass +class SshSettings: + """ + Transport-only config for the :class:`~linopy.remote.SSH` transport. + + Solver name and solver options come from :meth:`Model.solve` — + ``m.solve("gurobi", remote=SshSettings(hostname=...), presolve="on")``. + + Use ``setup_commands`` to prepare the remote shell before the solve — + e.g. activate a conda environment or set ``PATH``:: + + SshSettings(hostname=..., setup_commands=["conda activate linopy-env"]) + """ + + hostname: str + port: int = 22 + username: str | None = None + password: str | None = None + python_executable: str = "python" + python_file: str = "/tmp/linopy-execution.py" + model_unsolved_file: str = "/tmp/linopy-unsolved-model.nc" + model_solved_file: str = "/tmp/linopy-solved-model.nc" + setup_commands: list[str] = field(default_factory=list) + + @dataclass class RemoteHandler: """ Handler class for solving models on a remote machine via an SSH connection. + .. deprecated:: + ``RemoteHandler`` is the legacy low-level entry point and will be + removed in a future release. Prefer + ``Model.solve("gurobi", remote=SshSettings(hostname=...))`` or + instantiate :class:`SSH` directly. + The basic idea of the handler is to provide a workflow that: 1. defines a model on the local machine @@ -133,9 +166,20 @@ class RemoteHandler: model_unsolved_file: str = "/tmp/linopy-unsolved-model.nc" model_solved_file: str = "/tmp/linopy-solved-model.nc" + _internal: bool = field(default=False, repr=False) + def __post_init__(self) -> None: assert paramiko_present, "The required paramiko package is not installed." + if not self._internal: + warnings.warn( + "`RemoteHandler` is deprecated; use `SSH(settings, solver_name, " + "options)` from `linopy.remote` or `Model.solve(remote=SshSettings" + "(hostname=...))`. `RemoteHandler` will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) + if self.client is None: client = paramiko.SSHClient() client.load_system_host_keys() @@ -256,3 +300,82 @@ def solve_on_remote( self.sftp_client.remove(self.model_solved_file) return solved + + +@dataclass +class SSH: + """ + Remote handler that solves a linopy model on a remote machine over SSH. + + This is a standalone class — *not* a :class:`linopy.solvers.Solver` + subclass. It ships the model to a remote host and runs + ``read_netcdf(...).solve(solver_name=...)`` there, pulling the solved + netcdf back. + + Parameters + ---------- + settings : SshSettings + Connection + remote-execution paths. + solver_name : str + Solver to run on the remote (e.g. ``"gurobi"``). + options : dict, optional + Solver options passed through to the solver. + + Notes + ----- + Synchronous; unlike OETC the remote shell job is short-lived and + doesn't expose a useful submit/collect seam. + """ + + settings: SshSettings + solver_name: str + options: dict[str, Any] = field(default_factory=dict) + + _handler: "RemoteHandler | None" = field(init=False, default=None, repr=False) + _solved_model: Any = field(init=False, default=None, repr=False) + + @classmethod + def is_available(cls) -> bool: + """Return True iff paramiko is importable.""" + return paramiko_present + + def solve(self, model: "Model") -> Result: + """Ship the model, run the solver on the remote, return a Result.""" + from linopy.constants import Status + from linopy.remote._common import ( + _scatter_solution_from_solved_model, + _validate_inner_solver, + ) + + _validate_inner_solver(self.solver_name, model) + + if self._handler is None: + self._handler = RemoteHandler( + hostname=self.settings.hostname, + port=self.settings.port, + username=self.settings.username, + password=self.settings.password, + python_executable=self.settings.python_executable, + python_file=self.settings.python_file, + model_unsolved_file=self.settings.model_unsolved_file, + model_solved_file=self.settings.model_solved_file, + _internal=True, + ) + for cmd in self.settings.setup_commands: + self._handler.execute(cmd) + + solve_kwargs: dict[str, Any] = {"solver_name": self.solver_name} + if self.options: + solve_kwargs.update(self.options) + solved = self._handler.solve_on_remote(model, **solve_kwargs) + self._solved_model = solved + + status = Status.from_termination_condition(solved.termination_condition) + solution = _scatter_solution_from_solved_model( + model, solved, model._xCounter, model._cCounter + ) + return Result( + status=status, + solution=solution, + solver_name=self.solver_name, + ) diff --git a/linopy/solvers.py b/linopy/solvers.py index a28da898..6a6f23f0 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -515,11 +515,22 @@ def from_model( model: Model, io_api: str | None = None, options: dict[str, Any] | None = None, - **build_kwargs: Any, + **kwargs: Any, ) -> Solver: - """Instantiate and build the solver against ``model``.""" - instance = cls(model=model, io_api=io_api, options=options or {}) - instance._build(**build_kwargs) + """ + Instantiate and build the solver against ``model``. + + Any ``kwargs`` whose name matches an ``init=True`` dataclass field on + the subclass (e.g. ``settings`` on :class:`Oetc` / :class:`SSH`) are + forwarded to the constructor; the rest go to ``_build`` as + ``build_kwargs``. + """ + from dataclasses import fields + + field_names = {f.name for f in fields(cls) if f.init} + ctor_kw = {k: kwargs.pop(k) for k in list(kwargs) if k in field_names} + instance = cls(model=model, io_api=io_api, options=options or {}, **ctor_kw) + instance._build(**kwargs) return instance def _build(self, **build_kwargs: Any) -> None: diff --git a/pyproject.toml b/pyproject.toml index 67297677..ac916eb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ oetc = [ "google-cloud-storage", "requests", ] -remote = [ +ssh = [ "paramiko", ] docs = [ diff --git a/test/remote/test_oetc.py b/test/remote/test_oetc.py index 7b2d75f2..dd54b07d 100644 --- a/test/remote/test_oetc.py +++ b/test/remote/test_oetc.py @@ -1530,9 +1530,14 @@ def test_solve_on_oetc_file_upload( """Test solve_on_oetc method complete workflow""" # Setup mock_model = Mock() + mock_model._xCounter = 0 + mock_model._cCounter = 0 + mock_model.variables.items.return_value = [] + mock_model.constraints.items.return_value = [] mock_solved_model = Mock() mock_solved_model.status = "optimal" mock_solved_model.objective.value = 42.0 + mock_solved_model.termination_condition = "optimal" mock_temp_file = Mock() mock_temp_file.name = "/tmp/linopy-abc123.nc" @@ -1655,9 +1660,14 @@ def test_solve_on_oetc_with_job_submission( """Test solve_on_oetc method including job submission, waiting, and download""" # Setup mock_model = Mock() + mock_model._xCounter = 0 + mock_model._cCounter = 0 + mock_model.variables.items.return_value = [] + mock_model.constraints.items.return_value = [] mock_solved_model = Mock() mock_solved_model.status = "optimal" mock_solved_model.objective.value = 100.5 + mock_solved_model.termination_condition = "optimal" mock_temp_file = Mock() mock_temp_file.name = "/tmp/linopy-abc123.nc" diff --git a/test/remote/test_remotes.py b/test/remote/test_remotes.py new file mode 100644 index 00000000..5e9320b1 --- /dev/null +++ b/test/remote/test_remotes.py @@ -0,0 +1,373 @@ +""" +Tests for the standalone remote classes (``Oetc`` / ``SSH``) and the +``Model.solve(remote=)`` entry point. + +The deprecated ``OetcHandler`` / ``RemoteHandler`` are covered by +``test_oetc.py`` and ``test_ssh.py`` separately; this file focuses on +the *new* public surface and its deprecation warnings. +""" + +from __future__ import annotations + +import warnings +from typing import Any +from unittest.mock import MagicMock, patch + +import numpy as np +import pandas as pd +import pytest + +from linopy import Model +from linopy.constants import ( + Result, + Solution, + SolverReport, + Status, +) +from linopy.remote import ( + Oetc, + OetcCredentials, + OetcHandler, + OetcSettings, + RemoteHandler, + SshSettings, +) + +pytest.importorskip("paramiko") +from linopy.remote.ssh import SSH # noqa: E402 + +# --------------------------------------------------------------------------- +# Helpers + + +def _build_model() -> Model: + m = Model() + idx = pd.Index([0, 1, 2], name="i") + x = m.add_variables(lower=0, upper=1, coords=[idx], name="x") + m.add_constraints(x >= 0, name="c") + m.add_objective(1.0 * x.sum()) + return m + + +def _settings_oetc() -> OetcSettings: + return OetcSettings( + email="a@b.com", + password="pw", + name="test-job", + authentication_server_url="https://auth", + orchestrator_server_url="https://orch", + ) + + +def _settings_ssh() -> SshSettings: + return SshSettings(hostname="example.org", username="me") + + +def _fake_oetc_handler() -> MagicMock: + """A MagicMock(spec=OetcHandler) with the methods Oetc.upload/submit/collect call.""" + h = MagicMock(spec=OetcHandler) + h._upload_file_to_gcp = MagicMock(return_value="model.nc.gz") + h._submit_job_to_compute_service = MagicMock(return_value="job-uuid") + job_result = MagicMock() + job_result.output_files = [{"name": "result.nc.gz"}] + job_result.duration_in_seconds = 42 + h.wait_and_get_job_data = MagicMock(return_value=job_result) + h._download_file_from_gcp = MagicMock(return_value="/tmp/fake-result.nc") + return h + + +def _solved_model_like(m: Model) -> Model: + """Build a Model with the same labels as ``m`` plus dummy solution data.""" + solved = Model() + for name, var in m.variables.items(): + solved_var = solved.add_variables( + lower=var.lower, upper=var.upper, coords=var.coords, name=name + ) + solved_var.solution = solved_var.lower * 0 # zeros, real DataArray + for name, con in m.constraints.items(): + solved.add_constraints(con.lhs >= con.rhs, name=name) + solved.add_objective(m.objective.expression) + solved.objective._value = 0.0 + solved.termination_condition = "optimal" + solved.status = "ok" + return solved + + +# --------------------------------------------------------------------------- +# Oetc class + + +class TestOetcClass: + def test_solve_runs_upload_submit_collect( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() # bypass auth + + monkeypatch.setattr( + "linopy.remote.oetc.linopy.read_netcdf", + lambda path: _solved_model_like(m), + ) + + result = oetc.solve(m) + + assert isinstance(result, Result) + assert result.solver_name == "highs" + oetc._handler._upload_file_to_gcp.assert_called_once() + oetc._handler._submit_job_to_compute_service.assert_called_once() + oetc._handler.wait_and_get_job_data.assert_called_once_with("job-uuid") + oetc._handler._download_file_from_gcp.assert_called_once_with("result.nc.gz") + + def test_validates_unknown_solver_name(self) -> None: + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="not-a-solver") + oetc._handler = _fake_oetc_handler() + with pytest.raises(ValueError, match="Unknown solver"): + oetc.solve(m) + + def test_upload_submit_collect_separable( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The three-step lifecycle can be driven manually, e.g. for async work.""" + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() + monkeypatch.setattr( + "linopy.remote.oetc.linopy.read_netcdf", + lambda path: _solved_model_like(m), + ) + + oetc.upload(m) + assert oetc._input_file_name == "model.nc.gz" + assert oetc._handler._upload_file_to_gcp.call_count == 1 + + job_id = oetc.submit() + assert job_id == "job-uuid" + assert oetc._handler._submit_job_to_compute_service.call_count == 1 + + result = oetc.collect(m) + assert isinstance(result, Result) + assert oetc._handler.wait_and_get_job_data.call_count == 1 + + def test_submit_before_upload_raises(self) -> None: + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() + with pytest.raises(RuntimeError, match="upload"): + oetc.submit() + + def test_collect_before_submit_raises(self) -> None: + m = _build_model() + oetc = Oetc(settings=_settings_oetc(), solver_name="highs") + oetc._handler = _fake_oetc_handler() + with pytest.raises(RuntimeError, match="upload.*submit"): + oetc.collect(m) + + +# --------------------------------------------------------------------------- +# SSH class + + +class TestSSHClass: + def test_solve_runs_setup_commands_then_delegates(self) -> None: + m = _build_model() + ssh = SSH( + settings=SshSettings( + hostname="example.org", + setup_commands=["conda activate linopy-env", "export FOO=bar"], + ), + solver_name="highs", + ) + fake_handler = MagicMock(spec=RemoteHandler) + fake_handler.execute = MagicMock() + fake_handler.solve_on_remote = MagicMock(return_value=_solved_model_like(m)) + ssh._handler = fake_handler + + result = ssh.solve(m) + + assert isinstance(result, Result) + # solve_on_remote is the public surface from the deprecated handler + fake_handler.solve_on_remote.assert_called_once() + # setup_commands run only on first handler construction; here _handler + # was injected, so they shouldn't run automatically: + fake_handler.execute.assert_not_called() + + def test_setup_commands_run_when_handler_is_built_internally( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """First .solve() with a fresh SSH builds a RemoteHandler and runs setup.""" + m = _build_model() + ssh = SSH( + settings=SshSettings( + hostname="example.org", + setup_commands=["conda activate linopy-env"], + ), + solver_name="highs", + ) + + built: list[Any] = [] + + class FakeRemoteHandler: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + self.execute = MagicMock() + self.solve_on_remote = MagicMock(return_value=_solved_model_like(m)) + built.append(self) + + monkeypatch.setattr("linopy.remote.ssh.RemoteHandler", FakeRemoteHandler) + ssh.solve(m) + + assert len(built) == 1 + built[0].execute.assert_called_once_with("conda activate linopy-env") + assert built[0].kwargs.get("_internal") is True + + def test_validates_unknown_solver_name(self) -> None: + m = _build_model() + ssh = SSH(settings=_settings_ssh(), solver_name="not-a-solver") + ssh._handler = MagicMock(spec=RemoteHandler) + with pytest.raises(ValueError, match="Unknown solver"): + ssh.solve(m) + + +# --------------------------------------------------------------------------- +# Model.solve(remote=) end-to-end + + +class TestModelSolveRemote: + def test_oetc_settings_dispatches_to_oetc( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + captured: dict[str, Any] = {} + + def fake_solve(self: Oetc, model: Model) -> Result: + captured["solver_name"] = self.solver_name + captured["options"] = self.options + captured["instance"] = self + return Result( + status=Status.from_termination_condition("optimal"), + solution=Solution( + primal=np.zeros(model._xCounter, dtype=float), + dual=np.full(model._cCounter, np.nan, dtype=float), + objective=0.0, + ), + solver_name=self.solver_name, + report=SolverReport(runtime=1.0), + ) + + monkeypatch.setattr(Oetc, "solve", fake_solve) + + m.solve("gurobi", remote=_settings_oetc(), Method=2) + + assert captured["solver_name"] == "gurobi" + assert captured["options"] == {"Method": 2} + assert m.remote is captured["instance"] + assert m.solver is None # remote-solve clears any prior local solver + + def test_ssh_settings_dispatches_to_ssh( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + captured: dict[str, Any] = {} + + def fake_solve(self: SSH, model: Model) -> Result: + captured["solver_name"] = self.solver_name + captured["options"] = self.options + captured["instance"] = self + return Result( + status=Status.from_termination_condition("optimal"), + solution=Solution( + primal=np.zeros(model._xCounter, dtype=float), + dual=np.full(model._cCounter, np.nan, dtype=float), + objective=0.0, + ), + solver_name=self.solver_name, + ) + + monkeypatch.setattr(SSH, "solve", fake_solve) + + m.solve("highs", remote=_settings_ssh(), presolve="on") + + assert captured["solver_name"] == "highs" + assert captured["options"] == {"presolve": "on"} + assert m.remote is captured["instance"] + + +# --------------------------------------------------------------------------- +# Deprecation warnings + + +class TestDeprecations: + def test_oetc_credentials_construction_warns(self) -> None: + with pytest.warns(DeprecationWarning, match="OetcCredentials"): + OetcCredentials(email="a@b.com", password="pw") + + def test_oetc_settings_credentials_kwarg_carries_values_through(self) -> None: + # Constructing OetcCredentials warns (its own __post_init__). + with pytest.warns(DeprecationWarning, match="OetcCredentials"): + creds = OetcCredentials(email="a@b.com", password="pw") + + s = OetcSettings( + credentials=creds, + name="n", + authentication_server_url="https://a", + orchestrator_server_url="https://o", + ) + assert s.email == "a@b.com" + assert s.password == "pw" + # `credentials` is consumed and cleared. + assert s.credentials is None + + def test_oetc_settings_requires_email_and_password(self) -> None: + with pytest.raises(ValueError, match="email.*password"): + OetcSettings( + name="n", + authentication_server_url="https://a", + orchestrator_server_url="https://o", + ) + + def test_oetc_handler_construction_warns(self) -> None: + with ( + patch.object(OetcHandler, "_OetcHandler__sign_in"), + patch.object(OetcHandler, "_OetcHandler__get_cloud_provider_credentials"), + ): + with pytest.warns(DeprecationWarning, match="OetcHandler"): + OetcHandler(_settings_oetc()) + + def test_oetc_handler_internal_construction_silent(self) -> None: + with ( + patch.object(OetcHandler, "_OetcHandler__sign_in"), + patch.object(OetcHandler, "_OetcHandler__get_cloud_provider_credentials"), + ): + with warnings.catch_warnings(): + warnings.simplefilter("error") + OetcHandler(_settings_oetc(), _internal=True) + + def test_remote_handler_construction_warns( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + fake_client = MagicMock() + fake_client.invoke_shell.return_value.makefile.return_value = MagicMock() + fake_client.open_sftp.return_value = MagicMock() + + with pytest.warns(DeprecationWarning, match="RemoteHandler"): + RemoteHandler(hostname="x", client=fake_client) + + def test_remote_handler_internal_construction_silent(self) -> None: + fake_client = MagicMock() + fake_client.invoke_shell.return_value.makefile.return_value = MagicMock() + fake_client.open_sftp.return_value = MagicMock() + + with warnings.catch_warnings(): + warnings.simplefilter("error") + RemoteHandler(hostname="x", client=fake_client, _internal=True) + + def test_model_solve_remote_handler_warns( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + m = _build_model() + handler = MagicMock(spec=OetcHandler) + handler.settings = _settings_oetc() + handler.solve_on_oetc = MagicMock(return_value=_solved_model_like(m)) + with pytest.warns(DeprecationWarning, match="OetcHandler.*remote="): + m.solve(solver_name="highs", remote=handler) diff --git a/test/test_oetc_settings.py b/test/test_oetc_settings.py index 12deeb66..3206eaff 100644 --- a/test/test_oetc_settings.py +++ b/test/test_oetc_settings.py @@ -7,7 +7,6 @@ from linopy.remote.oetc import ( ComputeProvider, - OetcCredentials, OetcHandler, OetcSettings, ) @@ -48,8 +47,8 @@ def test_from_env_all_set(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OETC_DELETE_WORKER_ON_ERROR", "true") s = OetcSettings.from_env() - assert s.credentials.email == "test@example.com" - assert s.credentials.password == "secret" + assert s.email == "test@example.com" + assert s.password == "secret" assert s.name == "test-job" assert s.cpu_cores == 8 assert s.disk_space_gb == 20 @@ -62,7 +61,7 @@ def test_from_env_kwargs_override(monkeypatch: pytest.MonkeyPatch) -> None: _set_required_env(monkeypatch) s = OetcSettings.from_env(email="override@example.com") - assert s.credentials.email == "override@example.com" + assert s.email == "override@example.com" def test_from_env_missing_required(monkeypatch: pytest.MonkeyPatch) -> None: @@ -93,7 +92,7 @@ def test_from_env_partial_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OETC_ORCHESTRATOR_URL", "https://orch.example.com") s = OetcSettings.from_env(email="a@b.com", password="pw") - assert s.credentials.email == "a@b.com" + assert s.email == "a@b.com" assert s.name == "env-name" @@ -169,7 +168,8 @@ def _make_handler(settings: OetcSettings) -> OetcHandler: def _default_settings(**overrides: Any) -> OetcSettings: defaults: dict[str, Any] = dict( - credentials=OetcCredentials(email="a@b.com", password="pw"), + email="a@b.com", + password="pw", name="test", authentication_server_url="https://auth", orchestrator_server_url="https://orch", diff --git a/test/test_sos_reformulation.py b/test/test_sos_reformulation.py index 0e9dc9da..45a70694 100644 --- a/test/test_sos_reformulation.py +++ b/test/test_sos_reformulation.py @@ -6,16 +6,13 @@ import warnings from collections.abc import Callable from pathlib import Path -from typing import Literal, cast import numpy as np import pandas as pd import pytest -import xarray as xr from linopy import Model, Variable, available_solvers from linopy.constants import SOS_TYPE_ATTR -from linopy.remote import RemoteHandler from linopy.sos_reformulation import ( compute_big_m_values, reformulate_sos1, @@ -1188,64 +1185,57 @@ def _sos_model() -> Model: m.add_objective(x * np.array([1.0, 2.0, 3.0]), sense="max") return m - def _fake_handler( - self, observed: dict[str, object], tmp_path: Path - ) -> RemoteHandler: + @staticmethod + def _patch_ssh_solve( + monkeypatch: pytest.MonkeyPatch, + observed: dict[str, object], + tmp_path: Path, + ) -> None: """ - Non-OetcHandler stand-in with the SSH-shaped `solve_on_remote`. - - Records whether the model arrives in reformulated form, then runs - `model.to_netcdf(...)` and `read_netcdf(...)` (naturally — no - warning recording here, so we can observe at the call-site whether - Model.solve's suppression worked). + Replace ``linopy.remote.ssh.SSH.solve`` with a stub that records + whether the model arrives in reformulated form, exercises the + ``to_netcdf`` warning path, and returns a synthetic + :class:`Result` so ``Model.assign_result`` is exercised end to end. """ - from linopy.io import read_netcdf - from linopy.sos_reformulation import ( - sos_reformulation_context, - suppress_serialization_warning, - ) + from linopy.constants import Result, Solution, Status + from linopy.remote.ssh import SSH + + def fake_solve(self: SSH, model: Model) -> Result: + observed["state_active"] = model._sos_reformulation_state is not None + observed["solver_name_arg"] = self.solver_name + model.to_netcdf(tmp_path / "sent.nc") # triggers any to_netcdf warning + n_vars = model._xCounter + n_cons = model._cCounter + return Result( + status=Status.from_termination_condition("optimal"), + solution=Solution( + primal=np.zeros(n_vars, dtype=float), + dual=np.full(n_cons, np.nan, dtype=float), + objective=0.0, + ), + solver_name=self.solver_name, + ) + + monkeypatch.setattr(SSH, "solve", fake_solve) + + def test_remote_brackets_and_suppresses_warning( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from linopy.remote.ssh import SshSettings - class _Handler: - def solve_on_remote( - _self, - model: Model, - *, - reformulate_sos: bool | Literal["auto"] = False, - **kwargs: object, - ) -> Model: - solver_name = kwargs.get("solver_name") - assert solver_name is None or isinstance(solver_name, str) - with sos_reformulation_context( - model, solver_name, reformulate_sos - ) as applied: - observed["state_active"] = ( - model._sos_reformulation_state is not None - ) - observed["solver_name_arg"] = solver_name - with suppress_serialization_warning(active=applied): - model.to_netcdf(tmp_path / "sent.nc") - solved = read_netcdf(tmp_path / "sent.nc") - for _name, var in solved.variables.items(): - arr = np.zeros(var.labels.shape, dtype=float) - var.solution = xr.DataArray(arr, dims=var.labels.dims) - solved.objective.set_value(0.0) - solved.status = "ok" - solved.termination_condition = "optimal" - return solved - - return cast(RemoteHandler, _Handler()) - - def test_remote_brackets_and_suppresses_warning(self, tmp_path: Path) -> None: m = self._sos_model() observed: dict[str, object] = {} - handler = self._fake_handler(observed, tmp_path) + self._patch_ssh_solve(monkeypatch, observed, tmp_path) with warnings.catch_warnings(record=True) as captured: warnings.simplefilter("always") - m.solve(solver_name="highs", remote=handler, reformulate_sos=True) + m.solve( + solver_name="highs", + remote=SshSettings(hostname="ignored"), + reformulate_sos=True, + ) - # Reformulation was active when the handler ran (apply happened - # before the remote dispatch). + # Reformulation was active when the transport ran. assert observed["state_active"] is True assert observed["solver_name_arg"] == "highs" @@ -1258,26 +1248,38 @@ def test_remote_brackets_and_suppresses_warning(self, tmp_path: Path) -> None: assert "_sos_reform_x_y" not in m.variables def test_remote_skips_bracket_when_reformulate_sos_false( - self, tmp_path: Path + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + from linopy.remote.ssh import SshSettings + m = self._sos_model() observed: dict[str, object] = {} - handler = self._fake_handler(observed, tmp_path) + self._patch_ssh_solve(monkeypatch, observed, tmp_path) with warnings.catch_warnings(record=True) as captured: warnings.simplefilter("always") - m.solve(solver_name="highs", remote=handler, reformulate_sos=False) + m.solve( + solver_name="highs", + remote=SshSettings(hostname="ignored"), + reformulate_sos=False, + ) # No reformulation happened — model still has the original SOS var - # when the handler sees it, and to_netcdf never warns. + # when the transport sees it, and to_netcdf never warns. assert observed["state_active"] is False assert not any("active SOS reformulation" in str(w.message) for w in captured) assert m._sos_reformulation_state is None - def test_remote_auto_requires_solver_name_with_sos(self, tmp_path: Path) -> None: + def test_remote_auto_requires_solver_name_with_sos( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from linopy.remote.ssh import SshSettings + m = self._sos_model() observed: dict[str, object] = {} - handler = self._fake_handler(observed, tmp_path) + self._patch_ssh_solve(monkeypatch, observed, tmp_path) - with pytest.raises(ValueError, match="requires an explicit `solver_name`"): - m.solve(remote=handler, reformulate_sos="auto") + # Without an explicit solver_name, the transport dispatch refuses + # to run because there's no inner solver to ship. + with pytest.raises(ValueError, match="explicit `solver_name=`"): + m.solve(remote=SshSettings(hostname="ignored"), reformulate_sos="auto")