Pywr v2 export: fix for published schema, tests and CI for v1 and v2

.github/workflows/tests.yml

@@ -0,0 +1,64 @@
+name: Tests
+
+# hydra-pywr supports two Pywr runtimes that cannot share an environment (both install as the
+# `pywr` module), so the test suite runs once per runtime.
+#   pywr-v1: Pywr 1.x
+#   pywr-v2: the published Pywr 2.x pre-release wheel (no Rust toolchain needed)
+# All tests use temporary SQLite databases. No job has, or needs, access to a MySQL or live Hydra
+# database.
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+env:
+  # hydra-base from git so that fixes that have not been released yet are tested.
+  # TODO: use master once https://github.com/hydraplatform/hydra-base/pull/308 is merged.
+  HYDRA_BASE_REF: fix/template-cache-complete-typeattrs
+
+jobs:
+  test:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: pywr-v1
+            pywr_extra: v1
+            # Needs the hydropower nodes of the Pywr fork this model was written for; not
+            # available in Pywr 1.x from PyPI. Remove once it is ported (see tests/conftest.py).
+            pytest_args: >-
+              --deselect tests/test_model_results.py::TestResults::test_hydropower_results
+          - name: pywr-v2
+            pywr_extra: v2
+            pytest_args: ""
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          cache: pip
+
+      - name: System libraries (needed to build hydra-base's mysqlclient and pylibmc)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends build-essential pkg-config \
+            libmysqlclient-dev libmemcached-dev zlib1g-dev
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # hydra-base from git (see HYDRA_BASE_REF); the other git dependencies are declared
+          # in pyproject.toml.
+          python -m pip install \
+            "hydra-base @ git+https://github.com/hydraplatform/hydra-base@${HYDRA_BASE_REF}"
+          python -m pip install -e ".[dev,${{ matrix.pywr_extra }}]"
+
+      - name: Show installed Pywr
+        run: python -c "import pywr; print(pywr.__file__, getattr(pywr, '__version__', ''))"
+
+      - name: Run tests
+        run: python -m pytest tests -v ${{ matrix.pytest_args }}

hydra_pywr/cli.py

@@ -7,10 +7,7 @@
 
 from hydra_client.click import hydra_app
 
-from . import runner
-from . import exporter
-from . import importer
-
+from . import exporter_v2
 from . import utils
 
 
@@ -58,7 +55,7 @@ def cli(obj, username, password, hostname, session):
 @click.option('--rewrite-url-prefix', type=str, default=None)
 def import_json(obj, filename, project_id, template_id, projection, network_name, rewrite_url_prefix, *args):
     """ Import a Pywr JSON file into Hydra. """
-
+    from . import importer
     client = get_logged_in_client(obj)
 
     importer.import_json(client,
@@ -80,7 +77,7 @@ def import_json(obj, filename, project_id, template_id, projection, network_name
 @click.option('--scenario-name', type=str, default=None, help='Name for the new scenario (defaults to pywr metadata title)')
 def import_json_as_scenario(obj, filename, network_id, scenario_name, *args):
     """ Add a scenario to an existing Hydra Network from a Pywr JSON file """
-
+    from . import importer
     client = get_logged_in_client(obj)
 
     importer.import_json_as_scenario(client,
@@ -99,7 +96,7 @@ def import_json_as_scenario(obj, filename, network_id, scenario_name, *args):
 @click.option('--scenario-id', type=int, default=None, help='ID of an existing scenario to update instead of creating a new one')
 def import_json_as_scenario(obj, filename, network_id, scenario_id, scenario_name, *args):
     """ Add a scenario to an existing Hydra Network from a Pywr JSON file """
-
+    from . import importer
     client = get_logged_in_client(obj)
 
     importer.import_json_as_scenario(client,
@@ -120,6 +117,7 @@ def import_json_as_scenario(obj, filename, network_id, scenario_id, scenario_nam
 @click.option('--json-indent', type=int, default=2)
 def export_json(obj, data_dir, scenario_id, use_cache, json_sort_keys, json_indent):
     """ Export a Pywr JSON from Hydra. """
+    from . import exporter
     client = get_logged_in_client(obj)
     exporter.export_json(client,
                          data_dir,
@@ -129,6 +127,21 @@ def export_json(obj, data_dir, scenario_id, use_cache, json_sort_keys, json_inde
                          json_indent)
 
 
+@hydra_app(category='export', name='Export to Pywr v2 JSON')
+@cli.command(name='export-v2', context_settings=dict(
+    ignore_unknown_options=True,
+    allow_extra_args=True))
+@click.pass_obj
+@click.option('--data-dir', default='/tmp')
+@click.option('-s', '--scenario-id', type=int, default=None)
+@click.option('--json-sort-keys/--no-json-sort-keys', default=False)
+@click.option('--json-indent', type=int, default=2)
+def export_json_v2(obj, data_dir, scenario_id, json_sort_keys, json_indent):
+    """ Export a Pywr v2 JSON from Hydra. """
+    client = get_logged_in_client(obj)
+    exporter_v2.export_json_v2(client, data_dir, scenario_id, json_sort_keys, json_indent)
+
+
 @cli.command(name="run-file", context_settings=dict(
     ignore_unknown_options=True,
     allow_extra_args=True))
@@ -137,6 +150,7 @@ def export_json(obj, data_dir, scenario_id, use_cache, json_sort_keys, json_inde
 @click.option('--domain', type=str, default="water")
 @click.option('--output-file', type=str, default="output.csv")
 def run_file(obj, filename, domain, output_file):
+    from . import runner
     runner.run_file(filename, domain, output_file)
 
 @cli.command(name="purge-cache", context_settings=dict(
@@ -165,6 +179,7 @@ def purge_cache(cache_path):
 @click.option('--update', is_flag=True, default=False, help='Delete existing variable resource scenarios before saving new results to Hydra')
 def run(obj, scenario_id, template_id, domain, output_frequency, solver, data_dir, cache, disable_automatic_node_recorders, dry_run, update):
     """ Export, run and save a Pywr model from Hydra. """
+    from . import runner
     client = get_logged_in_client(obj)
 
     if scenario_id is None: