From 79655e41f1e8d971f33c4a6491f0f3d936fb7f80 Mon Sep 17 00:00:00 2001 From: James Tomlinson Date: Tue, 3 Nov 2026 21:01:28 +0000 Subject: [PATCH] feat(project): Add multi-models to project manifests. A multi-model composes several networks, linked by inter-network transfers, into a multi-network model. Its domain comes from a local model's base model and overrides, from a base model file, or is given inline. Each network starts from a network definition or from a model's network, may include further network set selections, and declares the transfers into it inline or in a transfers file. Models and multi-models share one namespace, so `compose` returns either kind, and `pywr-cli run-project` runs both. Transfer sources are checked against the multi-model's networks when the manifest is resolved, and the transferred components are checked once the networks are loaded. The example project now includes a neighbouring network that imports flow from the company network in a multi-model. --- examples/my-model/README.md | 26 +- examples/my-model/neighbour/base.json | 34 + examples/my-model/pywr-project.json | 42 ++ pywr-cli/src/main.rs | 221 ++++--- pywr-project/src/composition.rs | 239 ++++++- pywr-project/src/error.rs | 37 ++ pywr-project/src/lib.rs | 5 +- pywr-project/src/manifest/mod.rs | 21 +- pywr-project/src/manifest/v1.rs | 920 +++++++++++++++++++++++--- pywr-project/src/project.rs | 13 +- pywr-project/tests/example.rs | 9 + 11 files changed, 1355 insertions(+), 212 deletions(-) create mode 100644 examples/my-model/neighbour/base.json diff --git a/examples/my-model/README.md b/examples/my-model/README.md index 2935bc91..4fe47dc7 100644 --- a/examples/my-model/README.md +++ b/examples/my-model/README.md @@ -14,6 +14,8 @@ my-model/ │ └── inflows/ # network set `catchment::inflows`: one inflow per model │ ├── historical.json │ └── stochastic.json +├── neighbour/ # sub-network: a neighbouring supply system +│ └── base.json └── options/ # network set: optional supply schemes ├── option1.json └── option2.json @@ -26,17 +28,18 @@ for the base model node it connects to (`demand1` or `link1`) and adds a new sup ## Manifest -The manifest has four parts: +The manifest has five parts: -- **`sub_networks`** declare directories that hold part of the model, each with its own network - sets. The `catchment` sub-network has one set, `inflows`, referred to as `catchment::inflows`. - It sets `min` and `max` to 1, so every model that uses the catchment must select exactly one - inflow. +- **`sub_networks`** declare directories that hold part of the model, each with an optional base + network and its own network sets. The `catchment` sub-network has one set, `inflows`, referred + to as `catchment::inflows`. It sets `min` and `max` to 1, so every model that uses the + catchment must select exactly one inflow. The `neighbour` sub-network has only a base network. - **`network_sets`** name the project's own directories of fragments. - **`networks`** are reusable network definitions. Each selects files from the network sets. `historical` and `stochastic` name `catchment` as their `base`, which activates the - sub-network; a model cannot select from a sub-network's sets unless it is active. `option12` - extends `option1` and `option2` to combine them. + sub-network; a model cannot select from a sub-network's sets unless it is active. `neighbour` + is the `neighbour` sub-network's base network on its own. `option12` extends `option1` and + `option2` to combine them. - **`models`** are the runnable model definitions. Each starts from `base_model` and merges the networks it lists, in order. @@ -48,6 +51,13 @@ The manifest has four parts: | `baseline-option2` | `historical`, `option2` | Adds supply option 2 | | `baseline-option12` | `historical`, `option12` | Adds both supply options | +- **`multi_models`** are runnable multi-network models, whose networks run together and pass + values to each other through transfers. `joint-historical` takes its time domain from the + `baseline-historical` model and has two networks: + - `company`, the network of the `baseline-historical` model; + - `neighbour`, the `neighbour` network definition. Its `import` node's maximum flow is the + transfer `company-link1-flow`, which carries the flow through `link1` in `company`. + ## Running From the repository root: @@ -57,6 +67,8 @@ cargo run -p pywr-cli -- run-project examples/my-model/pywr-project.json \ --definition baseline-option1 --output-path /tmp/my-model ``` +`--definition` takes the name of a model or a multi-model. + The base model writes its results to `outputs.h5` in the output path. diff --git a/examples/my-model/neighbour/base.json b/examples/my-model/neighbour/base.json new file mode 100644 index 00000000..ea6f425d --- /dev/null +++ b/examples/my-model/neighbour/base.json @@ -0,0 +1,34 @@ +{ + "nodes": [ + { + "type": "Input", + "meta": { + "name": "import" + }, + "max_flow": { + "type": "InterNetworkTransfer", + "name": "company-link1-flow" + } + }, + { + "type": "Output", + "meta": { + "name": "demand2" + }, + "max_flow": { + "type": "Literal", + "value": 5.0 + }, + "cost": { + "type": "Literal", + "value": -10.0 + } + } + ], + "edges": [ + { + "from_node": "import", + "to_node": "demand2" + } + ] +} diff --git a/examples/my-model/pywr-project.json b/examples/my-model/pywr-project.json index 5f551a30..f6699b5b 100644 --- a/examples/my-model/pywr-project.json +++ b/examples/my-model/pywr-project.json @@ -2,6 +2,10 @@ "format": "v1", "base_model": "models/base.json", "sub_networks": [ + { + "name": "neighbour", + "base_network": "base.json" + }, { "name": "catchment", "network_sets": [ @@ -69,6 +73,11 @@ } ] }, + { + "name": "neighbour", + "description": "A neighbouring supply system.", + "base": "neighbour" + }, { "name": "option12", "description": "Both supply options.", @@ -112,5 +121,38 @@ "option12" ] } + ], + "multi_models": [ + { + "name": "joint-historical", + "domain": { + "model": "baseline-historical" + }, + "networks": [ + { + "name": "company", + "network": { + "model": "baseline-historical" + } + }, + { + "name": "neighbour", + "network": { + "network": "neighbour" + }, + "transfers": [ + { + "from_network": "company", + "metric": { + "type": "Node", + "name": "link1" + }, + "name": "company-link1-flow", + "initial_value": 0.0 + } + ] + } + ] + } ] } diff --git a/pywr-cli/src/main.rs b/pywr-cli/src/main.rs index df29f167..3ee2ac5b 100644 --- a/pywr-cli/src/main.rs +++ b/pywr-cli/src/main.rs @@ -149,7 +149,7 @@ enum Commands { RunProject { /// Path to Pywr project JSON. project: PathBuf, - /// Name of the definition to build. + /// Name of the model or multi-model definition to build. #[arg(short = 'd', long = "definition")] definition: String, /// Solver to use. @@ -562,6 +562,107 @@ fn run_server(mode: RunServerMode, socket_name: &str) -> Result<()> { Ok(()) } +/// Run `$model`, a `Model` or `MultiNetworkModel`, with the given solver settings. +macro_rules! run_with_solver { + ($model:expr, $solver:expr, $threads:expr, $ignore_feature_requirements:expr) => { + match *$solver { + Solver::Clp => { + let mut settings_builder = ClpSolverSettingsBuilder::default(); + if $threads > 1 { + settings_builder = settings_builder.parallel(); + settings_builder = settings_builder.threads($threads); + } + if $ignore_feature_requirements { + settings_builder = settings_builder.ignore_feature_requirements(); + } + let settings = settings_builder.build(); + $model.run(&settings) + } + #[cfg(feature = "cbc")] + Solver::Cbc => { + let mut settings_builder = CbcSolverSettingsBuilder::default(); + if $threads > 1 { + settings_builder = settings_builder.parallel(); + settings_builder = settings_builder.threads($threads); + } + if $ignore_feature_requirements { + settings_builder = settings_builder.ignore_feature_requirements(); + } + let settings = settings_builder.build(); + $model.run(&settings) + } + #[cfg(feature = "highs")] + Solver::Highs => { + let mut settings_builder = HighsSolverSettingsBuilder::default(); + if $threads > 1 { + settings_builder = settings_builder.parallel(); + settings_builder = settings_builder.threads($threads); + } + if $ignore_feature_requirements { + settings_builder = settings_builder.ignore_feature_requirements(); + } + let settings = settings_builder.build(); + $model.run(&settings) + } + #[cfg(feature = "ipm-ocl")] + Solver::CLIPMF32 => { + let mut settings_builder = ClIpmF32SettingsBuilder::default(); + if $threads > 1 { + settings_builder = settings_builder.parallel(); + settings_builder = settings_builder.threads($threads); + } + if $ignore_feature_requirements { + settings_builder = settings_builder.ignore_feature_requirements(); + } + + let settings = settings_builder.build(); + $model.run_multi_scenario(&settings) + } + #[cfg(feature = "ipm-ocl")] + Solver::CLIPMF64 => { + let mut settings_builder = ClIpmF64SettingsBuilder::default(); + if $threads > 1 { + settings_builder = settings_builder.parallel(); + settings_builder = settings_builder.threads($threads); + } + if $ignore_feature_requirements { + settings_builder = settings_builder.ignore_feature_requirements(); + } + + let settings = settings_builder.build(); + $model.run_multi_scenario(&settings) + } + #[cfg(feature = "ipm-simd")] + Solver::IpmSimd => { + let mut settings_builder = SimdIpmSolverSettingsBuilder::default(); + if $threads > 1 { + settings_builder = settings_builder.parallel(); + settings_builder = settings_builder.threads($threads); + } + if $ignore_feature_requirements { + settings_builder = settings_builder.ignore_feature_requirements(); + } + + let settings = settings_builder.build(); + $model.run_multi_scenario(&settings) + } + #[cfg(feature = "microlp")] + Solver::Microlp => { + let mut settings_builder = MicroLpSolverSettingsBuilder::default(); + if $threads > 1 { + settings_builder = settings_builder.parallel(); + settings_builder = settings_builder.threads($threads); + } + if $ignore_feature_requirements { + settings_builder = settings_builder.ignore_feature_requirements(); + } + let settings = settings_builder.build(); + $model.run(&settings) + } + } + }; +} + fn run_project( path: &Path, definition: &str, @@ -573,112 +674,28 @@ fn run_project( ) { let project = pywr_project::Project::open(path).unwrap(); let data_path = data_path.or_else(|| path.parent()); - - let composed_model = project.compose_model(definition).unwrap(); - let composed_schemas = composed_model.load().unwrap(); let options = pywr_schema::NetworkMergeOptions::default(); - let schema_v2 = composed_schemas.into_model_schema(&options).unwrap(); - let model_builder = schema_v2 - .create_model_builder(&FileSystem, data_path, output_path) - .unwrap(); - let model = model_builder.build().unwrap(); - - match *solver { - Solver::Clp => { - let mut settings_builder = ClpSolverSettingsBuilder::default(); - if threads > 1 { - settings_builder = settings_builder.parallel(); - settings_builder = settings_builder.threads(threads); - } - if ignore_feature_requirements { - settings_builder = settings_builder.ignore_feature_requirements(); - } - let settings = settings_builder.build(); - model.run(&settings) - } - #[cfg(feature = "cbc")] - Solver::Cbc => { - let mut settings_builder = CbcSolverSettingsBuilder::default(); - if threads > 1 { - settings_builder = settings_builder.parallel(); - settings_builder = settings_builder.threads(threads); - } - if ignore_feature_requirements { - settings_builder = settings_builder.ignore_feature_requirements(); - } - let settings = settings_builder.build(); - model.run(&settings) - } - #[cfg(feature = "highs")] - Solver::Highs => { - let mut settings_builder = HighsSolverSettingsBuilder::default(); - if threads > 1 { - settings_builder = settings_builder.parallel(); - settings_builder = settings_builder.threads(threads); - } - if ignore_feature_requirements { - settings_builder = settings_builder.ignore_feature_requirements(); - } - let settings = settings_builder.build(); - model.run(&settings) - } - #[cfg(feature = "ipm-ocl")] - Solver::CLIPMF32 => { - let mut settings_builder = ClIpmF32SettingsBuilder::default(); - if threads > 1 { - settings_builder = settings_builder.parallel(); - settings_builder = settings_builder.threads(threads); - } - if ignore_feature_requirements { - settings_builder = settings_builder.ignore_feature_requirements(); - } - let settings = settings_builder.build(); - model.run_multi_scenario(&settings) + match project.compose(definition).unwrap() { + pywr_project::Composed::Model(composed) => { + let schema = composed.load().unwrap().into_model_schema(&options).unwrap(); + let model = schema + .create_model_builder(&FileSystem, data_path, output_path) + .unwrap() + .build() + .unwrap(); + run_with_solver!(model, solver, threads, ignore_feature_requirements).unwrap(); } - #[cfg(feature = "ipm-ocl")] - Solver::CLIPMF64 => { - let mut settings_builder = ClIpmF64SettingsBuilder::default(); - if threads > 1 { - settings_builder = settings_builder.parallel(); - settings_builder = settings_builder.threads(threads); - } - if ignore_feature_requirements { - settings_builder = settings_builder.ignore_feature_requirements(); - } - - let settings = settings_builder.build(); - model.run_multi_scenario(&settings) - } - #[cfg(feature = "ipm-simd")] - Solver::IpmSimd => { - let mut settings_builder = SimdIpmSolverSettingsBuilder::default(); - if threads > 1 { - settings_builder = settings_builder.parallel(); - settings_builder = settings_builder.threads(threads); - } - if ignore_feature_requirements { - settings_builder = settings_builder.ignore_feature_requirements(); - } - - let settings = settings_builder.build(); - model.run_multi_scenario(&settings) - } - #[cfg(feature = "microlp")] - Solver::Microlp => { - let mut settings_builder = MicroLpSolverSettingsBuilder::default(); - if threads > 1 { - settings_builder = settings_builder.parallel(); - settings_builder = settings_builder.threads(threads); - } - if ignore_feature_requirements { - settings_builder = settings_builder.ignore_feature_requirements(); - } - let settings = settings_builder.build(); - model.run(&settings) + pywr_project::Composed::MultiModel(composed) => { + let schema = composed.load().unwrap().into_multi_model_schema(&options).unwrap(); + let model = schema + .create_model_builder(&FileSystem, data_path, output_path) + .unwrap() + .build() + .unwrap(); + run_with_solver!(model, solver, threads, ignore_feature_requirements).unwrap(); } } - .unwrap(); } // fn init_project_cli( diff --git a/pywr-project/src/composition.rs b/pywr-project/src/composition.rs index 4b85a52a..c132d76f 100644 --- a/pywr-project/src/composition.rs +++ b/pywr-project/src/composition.rs @@ -1,7 +1,11 @@ use crate::error::ComposeToSchemaError; use crate::manifest::DefinitionOverrides; use pywr_schema::meta::{ComponentMeta, ProvenanceSource}; -use pywr_schema::{ModelSchema, NetworkMergeOptions, NetworkSchema, NetworkSchemaReadError, VisitPaths}; +use pywr_schema::model::{Metadata, MultiNetworkEntry, MultiNetworkTransfer, ScenarioDomain, TimeDomain}; +use pywr_schema::{ + ModelSchema, MultiNetworkModelSchema, NetworkMergeOptions, NetworkSchema, NetworkSchemaReadError, NetworkSchemaRef, + Reference, VisitPaths, VisitReferences, +}; use relative_path::RelativePathBuf; use std::path::{Path, PathBuf}; @@ -184,6 +188,8 @@ fn rebase_data_paths(network: &mut NetworkSchema, data_dir: &Path) { /// A composed network that combines the networks selected by a network definition. pub struct ComposedNetworkSchemas { name: String, + /// The network of a model definition's base model, which the includes are merged into. + base_network: Option, includes: Vec, } @@ -196,23 +202,35 @@ impl ComposedNetworkSchemas { &self.includes } - /// Merge the included networks in order, starting from an empty network. + /// Merge the included networks in order, starting from the base network if there is one and + /// an empty network otherwise. pub fn into_network_schema(self, options: &NetworkMergeOptions) -> Result { - let mut network = NetworkSchema::default(); + let mut network = self.base_network.unwrap_or_default(); merge_includes(&mut network, self.includes, options)?; Ok(network) } } -/// A composed network that combines the networks selected by a network definition. +/// A composed network that combines the networks selected by a network definition, or the +/// network part of a model definition. pub struct ComposedNetwork { name: String, + /// A model definition's base model, whose network is the starting point. + base_model: Option<(PathBuf, ProvenanceSource)>, includes: Vec, } impl ComposedNetwork { - pub(crate) fn new(name: String, includes: Vec) -> Self { - Self { name, includes } + pub(crate) fn new( + name: String, + base_model: Option<(PathBuf, ProvenanceSource)>, + includes: Vec, + ) -> Self { + Self { + name, + base_model, + includes, + } } pub fn name(&self) -> &str { @@ -221,17 +239,224 @@ impl ComposedNetwork { /// Deserialize the included networks from their paths, returning a [`ComposedNetworkSchemas`] instance. pub fn load(&self) -> Result { + let base_network = match &self.base_model { + Some((path, source)) => { + let mut network = ModelSchema::from_path(path)?.network; + network.set_provenance(source.clone()); + Some(network) + } + None => None, + }; Ok(ComposedNetworkSchemas { name: self.name.clone(), + base_network, includes: load_includes(&self.includes)?, }) } pub fn all_paths(&self) -> Vec { - self.includes.iter().map(|n| n.path.clone()).collect() + self.base_model + .iter() + .map(|(path, _)| path.clone()) + .chain(self.includes.iter().map(|n| n.path.clone())) + .collect() } } +/// Where a composed multi-model takes its time domain, scenarios and metadata from. +pub enum ComposedDomain { + /// A base model file, whose network is ignored. + BaseModel(PathBuf), + Inline { + time: TimeDomain, + scenarios: Option, + }, +} + +/// One network of a composed multi-model, with the transfers into it. +pub struct ComposedMultiModelNetwork { + pub network: ComposedNetwork, + pub transfers: Vec, +} + +/// A composed multi-network model: several composed networks, linked by transfers, that share a +/// time domain and scenarios. +pub struct ComposedMultiModel { + name: String, + domain: ComposedDomain, + /// Overrides applied to the domain, in order. + overrides: Vec, + networks: Vec, +} + +impl ComposedMultiModel { + pub(crate) fn new( + name: String, + domain: ComposedDomain, + overrides: Vec, + networks: Vec, + ) -> Self { + Self { + name, + domain, + overrides, + networks, + } + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn networks(&self) -> &[ComposedMultiModelNetwork] { + &self.networks + } + + /// Deserialize the domain and networks from their paths, returning a [`ComposedMultiModelSchemas`] instance. + pub fn load(&self) -> Result { + let (metadata, mut time, mut scenarios) = match &self.domain { + ComposedDomain::BaseModel(path) => { + let model = ModelSchema::from_path(path)?; + (model.metadata, model.time, model.scenarios) + } + ComposedDomain::Inline { time, scenarios } => (Metadata::default(), time.clone(), scenarios.clone()), + }; + for overrides in &self.overrides { + if let Some(overridden) = &overrides.time { + time = overridden.clone(); + } + if let Some(overridden) = &overrides.scenarios { + scenarios = Some(overridden.clone()); + } + } + let networks = self + .networks + .iter() + .map(|entry| Ok((entry.network.load()?, entry.transfers.clone()))) + .collect::>()?; + Ok(ComposedMultiModelSchemas { + metadata: Metadata { + title: self.name.clone(), + ..metadata + }, + time, + scenarios, + networks, + }) + } + + pub fn all_paths(&self) -> Vec { + let domain = match &self.domain { + ComposedDomain::BaseModel(path) => Some(path.clone()), + ComposedDomain::Inline { .. } => None, + }; + domain + .into_iter() + .chain(self.networks.iter().flat_map(|entry| entry.network.all_paths())) + .collect() + } +} + +/// A composed multi-network model with its networks loaded but not yet merged. +pub struct ComposedMultiModelSchemas { + metadata: Metadata, + time: TimeDomain, + scenarios: Option, + networks: Vec<(ComposedNetworkSchemas, Vec)>, +} + +impl ComposedMultiModelSchemas { + /// Merge each network and return a [`MultiNetworkModelSchema`] with every network inline. + /// + /// Fails if a network has unresolved placeholders, or a transfer reads a node, virtual node + /// or parameter that its source network does not have. + pub fn into_multi_model_schema( + self, + options: &NetworkMergeOptions, + ) -> Result { + let mut networks = Vec::with_capacity(self.networks.len()); + let mut placeholders = Vec::new(); + for (schemas, transfers) in self.networks { + let name = schemas.name().to_string(); + let network = schemas.into_network_schema(options)?; + placeholders.extend( + unresolved_placeholders(&network) + .into_iter() + .map(|placeholder| format!("{placeholder} in network `{name}`")), + ); + networks.push(MultiNetworkEntry { + name, + network: NetworkSchemaRef::Inline(network), + transfers, + }); + } + if !placeholders.is_empty() { + return Err(ComposeToSchemaError::UnresolvedPlaceholders { placeholders }); + } + check_transfer_components(&networks)?; + + Ok(MultiNetworkModelSchema { + metadata: self.metadata, + time: self.time, + scenarios: self.scenarios, + networks, + }) + } +} + +/// Check that each transfer reads a node, virtual node or parameter that its source network has. +fn check_transfer_components(networks: &[MultiNetworkEntry]) -> Result<(), ComposeToSchemaError> { + let inline = |name: &str| { + networks + .iter() + .find(|entry| entry.name == name) + .and_then(|entry| match &entry.network { + NetworkSchemaRef::Inline(network) => Some(network), + NetworkSchemaRef::Path(_) => None, + }) + }; + for entry in networks { + for transfer in &entry.transfers { + let Some(source) = inline(&transfer.from_network) else { + continue; + }; + let mut missing = None; + transfer.metric.visit_references(&mut |reference| { + if missing.is_some() { + return; + } + missing = match reference { + Reference::Node { name, .. } if source.get_node_by_name(name).is_none() => { + Some(format!("the node `{name}`")) + } + Reference::VirtualNode { name, .. } if source.get_virtual_node_by_name(name).is_none() => { + Some(format!("the virtual node `{name}`")) + } + Reference::Parameter { name, .. } if source.get_parameter_by_name(name).is_none() => { + Some(format!("the parameter `{name}`")) + } + _ => None, + }; + }); + if let Some(component) = missing { + return Err(ComposeToSchemaError::TransferComponentNotFound { + network: entry.name.clone(), + transfer: transfer.name.clone(), + from_network: transfer.from_network.clone(), + component, + }); + } + } + } + Ok(()) +} + +/// A composed model or multi-model, as named by a definition. +pub enum Composed { + Model(ComposedModel), + MultiModel(ComposedMultiModel), +} + /// How an included network is adjusted as it is loaded. #[derive(Clone, Debug, Default)] pub struct LoadContext { diff --git a/pywr-project/src/error.rs b/pywr-project/src/error.rs index 2f854513..a86a9b17 100644 --- a/pywr-project/src/error.rs +++ b/pywr-project/src/error.rs @@ -34,6 +34,15 @@ pub enum ComposeToSchemaError { NetworkMerge(#[from] NetworkMergeError), #[error("The composed model has unresolved placeholders: {}.", .placeholders.join(", "))] UnresolvedPlaceholders { placeholders: Vec }, + #[error( + "The transfer `{transfer}` into `{network}` reads {component} from `{from_network}`, which does not exist there." + )] + TransferComponentNotFound { + network: String, + transfer: String, + from_network: String, + component: String, + }, } /// Errors that may occur during validation of a project manifest, or during resolution of a @@ -72,6 +81,20 @@ pub enum ManifestResolutionError { BaseNetworkNotFound { sub_network: String, path: PathBuf }, #[error("The base network `{}` for sub-network `{sub_network}` is not a regular file.", .path.display())] BaseNetworkNotAFile { sub_network: String, path: PathBuf }, + #[error("The transfers file `{}` was not found.", .path.display())] + TransfersFileNotFound { path: PathBuf }, + #[error("The transfers file `{}` could not be read.", .path.display())] + TransfersFileRead { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("The transfers file `{}` does not contain a valid list of transfers.", .path.display())] + TransfersFileJson { + path: PathBuf, + #[source] + source: serde_json::Error, + }, #[error("The path `{}` for {field} could not be canonicalized.", .path.display())] UnableToCanonicalizePath { field: String, @@ -116,6 +139,20 @@ pub enum ComposeModelError { NetworkCycle { cycle: Vec }, #[error("The model definition `{definition}` has no base model.")] BaseModelMissing { definition: String }, + #[error("The definition `{definition}` refers to the model definition `{model}`, which does not exist.")] + ModelNotFound { definition: String, model: String }, + #[error("The multi-model `{definition}` has more than one network named `{network}`.")] + DuplicateMultiModelNetwork { definition: String, network: String }, + #[error("The transfer `{transfer}` into `{network}` is from the network `{from_network}`, which does not exist.")] + TransferSourceNotFound { + network: String, + transfer: String, + from_network: String, + }, + #[error("The transfer `{transfer}` into `{network}` is from the same network.")] + TransferFromSameNetwork { network: String, transfer: String }, + #[error("The name `{transfer}` is used by more than one transfer into `{network}`.")] + DuplicateTransfer { network: String, transfer: String }, #[error("The network set `{set}` is selected more than once, but each set may be selected at most once.")] DuplicateSelection { set: String }, #[error("The network set `{set}` has `min_files` {min_files}, which exceeds `max_files` {max_files}.")] diff --git a/pywr-project/src/lib.rs b/pywr-project/src/lib.rs index 1d2bbf17..7d00a473 100644 --- a/pywr-project/src/lib.rs +++ b/pywr-project/src/lib.rs @@ -4,8 +4,9 @@ mod manifest; mod project; pub use composition::{ - ComposedModel, ComposedModelBuilder, ComposedModelNetworkSchema, ComposedModelSchemas, ComposedNetwork, - ComposedNetworkPath, ComposedNetworkSchemas, LoadContext, PositionOffset, + Composed, ComposedDomain, ComposedModel, ComposedModelBuilder, ComposedModelNetworkSchema, ComposedModelSchemas, + ComposedMultiModel, ComposedMultiModelNetwork, ComposedMultiModelSchemas, ComposedNetwork, ComposedNetworkPath, + ComposedNetworkSchemas, LoadContext, PositionOffset, }; pub use error::{ ComposeModelError, ComposeToSchemaError, ManifestResolutionError, ProjectError, ProjectManifestReadError, diff --git a/pywr-project/src/manifest/mod.rs b/pywr-project/src/manifest/mod.rs index a41df2ca..b44870b0 100644 --- a/pywr-project/src/manifest/mod.rs +++ b/pywr-project/src/manifest/mod.rs @@ -1,6 +1,6 @@ pub mod v1; -use crate::composition::{ComposedModel, ComposedNetwork}; +use crate::composition::{Composed, ComposedModel, ComposedMultiModel, ComposedNetwork}; use crate::error::{ComposeModelError, ProjectManifestReadError}; use pywr_schema::model::{ScenarioDomain, TimeDomain}; use schemars::JsonSchema; @@ -66,12 +66,29 @@ impl ProjectManifest { } } - pub fn compose(&self, root: &Path, definition_name: &str) -> Result { + /// Compose the model or multi-model definition `definition_name`. + pub fn compose(&self, root: &Path, definition_name: &str) -> Result { + match self { + ProjectManifest::V1(manifest) => manifest.compose(root, definition_name), + } + } + + pub fn compose_model(&self, root: &Path, definition_name: &str) -> Result { match self { ProjectManifest::V1(manifest) => manifest.compose_model(root, definition_name), } } + pub fn compose_multi_model( + &self, + root: &Path, + definition_name: &str, + ) -> Result { + match self { + ProjectManifest::V1(manifest) => manifest.compose_multi_model(root, definition_name), + } + } + pub fn compose_network(&self, root: &Path, network_name: &str) -> Result { match self { ProjectManifest::V1(manifest) => manifest.compose_network(root, network_name), diff --git a/pywr-project/src/manifest/v1.rs b/pywr-project/src/manifest/v1.rs index 65f632b9..9d65cc34 100644 --- a/pywr-project/src/manifest/v1.rs +++ b/pywr-project/src/manifest/v1.rs @@ -1,9 +1,11 @@ use crate::composition::{ - ComposedModel, ComposedModelBuilder, ComposedNetwork, ComposedNetworkPath, LoadContext, PositionOffset, + Composed, ComposedDomain, ComposedModel, ComposedModelBuilder, ComposedMultiModel, ComposedMultiModelNetwork, + ComposedNetwork, ComposedNetworkPath, LoadContext, PositionOffset, }; use crate::error::{ComposeModelError, ManifestResolutionError}; use crate::manifest::DefinitionOverrides; use pywr_schema::meta::ProvenanceSource; +use pywr_schema::model::{MultiNetworkTransfer, ScenarioDomain, TimeDomain}; use relative_path::{RelativePath, RelativePathBuf}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -92,6 +94,28 @@ pub enum ProjectManifestProblem { "The model definition `{definition}` has no base model; give one in the definition or at the top of the manifest." )] BaseModelMissing { definition: String }, + #[error("The definition `{definition}` refers to the model definition `{model}`, which does not exist.")] + ModelNotFound { definition: String, model: String }, + #[error("The multi-model `{definition}` has {count} networks named `{network}`, but each name must be unique.")] + DuplicateMultiModelNetwork { + definition: String, + network: String, + count: usize, + }, + #[error("The transfer `{transfer}` into `{network}` is from the network `{from_network}`, which does not exist.")] + TransferSourceNotFound { + network: String, + transfer: String, + from_network: String, + }, + #[error("The transfer `{transfer}` into `{network}` is from the same network.")] + TransferFromSameNetwork { network: String, transfer: String }, + #[error("The name `{transfer}` is used by {count} transfers into `{network}`, but each name must be unique.")] + DuplicateTransfer { + network: String, + transfer: String, + count: usize, + }, #[error( "The definition `{definition}` selects the network set `{set}` {count} times, but it may select each set at most once." )] @@ -178,6 +202,9 @@ pub struct ProjectManifest { /// Named model definitions. #[serde(default)] pub models: Vec, + /// Named multi-network model definitions. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub multi_models: Vec, } impl ProjectManifest { @@ -186,34 +213,39 @@ impl ProjectManifest { let mut problems = self.manifest_problems(); let base_model = self.resolve_base_model(root, &mut problems); let project = self.resolve_project(root, &mut problems); - let mut networks = NetworkExpander::new(self, &project); + let mut resolver = Resolver::new(self, &project, root, &base_model); for model in &self.models { - model.resolve(root, &base_model, &mut networks, &mut problems); + resolver.resolve_model(model, &mut problems); + } + for multi_model in &self.multi_models { + multi_model.resolve(&mut resolver, &mut problems); } // Network definitions that no model uses are still validated; each is reported once. // Sub-network activation is not checked here, because a model may activate it instead. for network in &self.networks { - networks.expand(&network.name, &mut problems); + resolver.expand(&network.name, &mut problems); } into_result(problems) } - /// Validate a model definition against the project manifest. + /// Validate a model or multi-model definition against the project manifest. pub fn validate_model(&self, root: &Path, definition_name: &str) -> Result<(), ProjectManifestValidationError> { - let Some(model) = self.find_model(definition_name) else { + let model = self.find_model(definition_name); + let multi_model = self.find_multi_model(definition_name); + if model.is_none() && multi_model.is_none() { return into_result(vec![ProjectManifestProblem::DefinitionNotFound { definition: definition_name.to_string(), }]); - }; + } let mut problems = self.manifest_problems(); let base_model = self.resolve_base_model(root, &mut problems); let project = self.resolve_project(root, &mut problems); - model.resolve( - root, - &base_model, - &mut NetworkExpander::new(self, &project), - &mut problems, - ); + let mut resolver = Resolver::new(self, &project, root, &base_model); + if let Some(model) = model { + resolver.resolve_model(model, &mut problems); + } else if let Some(multi_model) = multi_model { + multi_model.resolve(&mut resolver, &mut problems); + } into_result(problems) } @@ -226,11 +258,22 @@ impl ProjectManifest { } let mut problems = self.manifest_problems(); let project = self.resolve_project(root, &mut problems); - let expansion = NetworkExpander::new(self, &project).expand(network_name, &mut problems); + let expansion = + Resolver::new(self, &project, root, &ProjectBaseModel::NotGiven).expand(network_name, &mut problems); check_active(network_name, &expansion, &mut problems); into_result(problems) } + /// Compose the model or multi-model definition `definition_name`. + pub fn compose(&self, root: &Path, definition_name: &str) -> Result { + if self.find_multi_model(definition_name).is_some() { + self.compose_multi_model(root, definition_name) + .map(Composed::MultiModel) + } else { + self.compose_model(root, definition_name).map(Composed::Model) + } + } + /// Compose a model from its base model and the specified model definition. /// /// This resolves the manifest in the same way as [`Self::validate_model`], and returns the @@ -245,12 +288,7 @@ impl ProjectManifest { }; let base_model = self.resolve_base_model(root, &mut problems); let project = self.resolve_project(root, &mut problems); - let resolved = model.resolve( - root, - &base_model, - &mut NetworkExpander::new(self, &project), - &mut problems, - ); + let resolved = Resolver::new(self, &project, root, &base_model).resolve_model(model, &mut problems); match (problems.into_iter().next(), resolved.base_model) { (Some(problem), _) => Err(problem.into()), @@ -268,6 +306,38 @@ impl ProjectManifest { } } + /// Compose a multi-network model from the specified multi-model definition. + /// + /// This resolves the manifest in the same way as [`Self::validate_model`], and returns the + /// first problem found as an error. + pub fn compose_multi_model( + &self, + root: &Path, + definition_name: &str, + ) -> Result { + let mut problems = self.manifest_problems(); + let Some(multi_model) = self.find_multi_model(definition_name) else { + problems.push(ProjectManifestProblem::DefinitionNotFound { + definition: definition_name.to_string(), + }); + return Err(problems.swap_remove(0).into()); + }; + let base_model = self.resolve_base_model(root, &mut problems); + let project = self.resolve_project(root, &mut problems); + let resolved = multi_model.resolve(&mut Resolver::new(self, &project, root, &base_model), &mut problems); + + match (problems.into_iter().next(), resolved.domain) { + (Some(problem), _) => Err(problem.into()), + (None, Some(domain)) => Ok(ComposedMultiModel::new( + multi_model.name.clone(), + domain, + resolved.overrides, + resolved.networks, + )), + (None, None) => unreachable!("a domain that fails to resolve is always reported as a problem"), + } + } + /// Compose a network from the specified network definition. /// /// This resolves the manifest in the same way as [`Self::validate_network`], and returns the @@ -281,13 +351,15 @@ impl ProjectManifest { return Err(problems.swap_remove(0).into()); } let project = self.resolve_project(root, &mut problems); - let expansion = NetworkExpander::new(self, &project).expand(network_name, &mut problems); + let expansion = + Resolver::new(self, &project, root, &ProjectBaseModel::NotGiven).expand(network_name, &mut problems); check_active(network_name, &expansion, &mut problems); match problems.into_iter().next() { Some(problem) => Err(problem.into()), None => Ok(ComposedNetwork::new( network_name.to_string(), + None, expansion.files.into_iter().map(SelectedFile::into_composed).collect(), )), } @@ -301,12 +373,23 @@ impl ProjectManifest { self.networks.iter().find(|network| network.name == name) } + fn find_multi_model(&self, name: &str) -> Option<&MultiModelDefinition> { + self.multi_models.iter().find(|multi_model| multi_model.name == name) + } + /// Problems in the manifest itself, found without touching the filesystem. fn manifest_problems(&self) -> Vec { let mut problems = Vec::new(); + // Models and multi-models share one namespace, so a definition name says what to run. + let definitions: Vec<&String> = self + .models + .iter() + .map(|model| &model.name) + .chain(self.multi_models.iter().map(|multi_model| &multi_model.name)) + .collect(); add_duplicate_problems( - &self.models, - |model| &model.name, + &definitions, + |name| name, |name, count| ProjectManifestProblem::DuplicateDefinition { definition: name, count, @@ -361,6 +444,11 @@ impl ProjectManifest { .models .iter() .map(|model| ("model definition", &model.name)) + .chain( + self.multi_models + .iter() + .map(|multi_model| ("multi-model definition", &multi_model.name)), + ) .chain( self.networks .iter() @@ -585,26 +673,132 @@ fn check_active(definition: &str, expansion: &Expansion<'_>, problems: &mut Vec< } } -/// Expands network definitions into their selected files, remembering each expansion so that a -/// definition's problems are reported once however many definitions refer to it. -struct NetworkExpander<'a> { +/// Resolves model and network definitions into their selected files, remembering each result +/// so that a definition's problems are reported once however many definitions refer to it. +struct Resolver<'a> { manifest: &'a ProjectManifest, project: &'a ResolvedProject, + root: &'a Path, + project_base_model: &'a ProjectBaseModel, expanded: HashMap<&'a str, Expansion<'a>>, - /// The definitions currently being expanded, outermost first, used to detect cycles. + expanded_models: HashMap<&'a str, Expansion<'a>>, + model_base_models: HashMap<&'a str, Option>, + /// The network definitions currently being expanded, outermost first, used to detect cycles. stack: Vec<&'a str>, } -impl<'a> NetworkExpander<'a> { - fn new(manifest: &'a ProjectManifest, project: &'a ResolvedProject) -> Self { +impl<'a> Resolver<'a> { + fn new( + manifest: &'a ProjectManifest, + project: &'a ResolvedProject, + root: &'a Path, + project_base_model: &'a ProjectBaseModel, + ) -> Self { Self { manifest, project, + root, + project_base_model, expanded: HashMap::new(), + expanded_models: HashMap::new(), + model_base_models: HashMap::new(), stack: Vec::new(), } } + /// Resolve a model definition's base model and files, checking them as a whole model. + fn resolve_model( + &mut self, + model: &'a ModelDefinition, + problems: &mut Vec, + ) -> ResolvedModel { + let base_model = self.model_base_model(model, problems); + let expansion = self.expand_model(model, problems); + check_composition(&model.name, &expansion, self.project, problems); + ResolvedModel { + base_model, + includes: expansion.files.into_iter().map(SelectedFile::into_composed).collect(), + } + } + + /// The base model of a model definition: its own, or else the project's. + fn model_base_model( + &mut self, + model: &'a ModelDefinition, + problems: &mut Vec, + ) -> Option { + if let Some(base_model) = self.model_base_models.get(model.name.as_str()) { + return base_model.clone(); + } + let base_model = match (&model.base_model, self.project_base_model) { + (Some(base_model), _) => match resolve_base_model(self.root, base_model) { + Ok(base_model) => Some(base_model), + Err(error) => { + problems.push(error.into()); + None + } + }, + (None, ProjectBaseModel::Resolved(base_model)) => Some(base_model.clone()), + (None, ProjectBaseModel::Unresolved) => None, + (None, ProjectBaseModel::NotGiven) => { + problems.push(ProjectManifestProblem::BaseModelMissing { + definition: model.name.clone(), + }); + None + } + }; + self.model_base_models.insert(&model.name, base_model.clone()); + base_model + } + + /// Expand a model definition into the files merged after its base model, in merge order: + /// the base networks of the sub-networks it activates, its networks, then its own selections. + fn expand_model( + &mut self, + model: &'a ModelDefinition, + problems: &mut Vec, + ) -> Expansion<'a> { + if let Some(expansion) = self.expanded_models.get(model.name.as_str()) { + return expansion.clone(); + } + let project = self.project; + let mut expansion = Expansion::default(); + for name in &model.sub_networks { + if let Some(sub_network) = project.referred_sub_network(&model.name, name, problems) { + expansion.active_sub_networks.push(&sub_network.name); + } + } + // The base networks of the sub-networks activated here come straight after the base model. + for sub_network in &project.sub_networks { + if let Some(file) = &sub_network.base_network + && expansion.active_sub_networks.contains(&sub_network.name.as_str()) + { + expansion.files.push(SelectedFile { + file, + set: None, + position_offset: None, + network: None, + }); + } + } + for network in &model.networks { + if self.manifest.find_network(network).is_none() { + problems.push(ProjectManifestProblem::NetworkNotFound { + definition: model.name.clone(), + network: network.clone(), + }); + } + let network = self.expand(network, problems); + expansion.extend(network); + } + expansion + .files + .extend(resolve_includes(&model.name, None, &model.include, project, problems)); + expansion.dedup(); + self.expanded_models.insert(&model.name, expansion.clone()); + expansion + } + /// Expand the network definition `name` into its files, in merge order: its base /// sub-network's base network, those of each definition it extends, then its own selections. /// @@ -761,6 +955,25 @@ impl From for ComposeModelError { }, ProjectManifestProblem::NetworkCycle { cycle } => Self::NetworkCycle { cycle }, ProjectManifestProblem::BaseModelMissing { definition } => Self::BaseModelMissing { definition }, + ProjectManifestProblem::ModelNotFound { definition, model } => Self::ModelNotFound { definition, model }, + ProjectManifestProblem::DuplicateMultiModelNetwork { + definition, network, .. + } => Self::DuplicateMultiModelNetwork { definition, network }, + ProjectManifestProblem::TransferSourceNotFound { + network, + transfer, + from_network, + } => Self::TransferSourceNotFound { + network, + transfer, + from_network, + }, + ProjectManifestProblem::TransferFromSameNetwork { network, transfer } => { + Self::TransferFromSameNetwork { network, transfer } + } + ProjectManifestProblem::DuplicateTransfer { network, transfer, .. } => { + Self::DuplicateTransfer { network, transfer } + } ProjectManifestProblem::DuplicateSet { set, .. } => Self::DuplicateSelection { set }, ProjectManifestProblem::SetNotFound { set, .. } => Self::SetNotFound { set }, ProjectManifestProblem::FileNotFound { set, file, .. } => Self::FileNotFound { set, file }, @@ -873,92 +1086,324 @@ struct ResolvedModel { includes: Vec, } -impl ModelDefinition { - /// Resolve this definition's base model and selections. +/// Check an expansion as the complete network of `definition`: every sub-network it selects +/// from must be active, and every network set's limits must hold. +fn check_composition( + definition: &str, + expansion: &Expansion<'_>, + project: &ResolvedProject, + problems: &mut Vec, +) { + check_active(definition, expansion, problems); + + // Set constraints apply to everything selected from a set, by any route. A sub-network's + // sets are only constrained when the sub-network is active. + for set in &project.sets { + if let Some(sub_network) = &set.sub_network + && !expansion.active_sub_networks.contains(&sub_network.as_str()) + { + continue; + } + let count = expansion + .files + .iter() + .filter(|selected| selected.set.is_some_and(|selected_set| std::ptr::eq(selected_set, set))) + .count(); + validate_constraints(definition, set, count, problems); + } +} + +/// A named multi-network model: several networks, linked by transfers, that share a time domain +/// and scenarios. +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MultiModelDefinition { + pub name: String, + /// Where the time domain, scenarios and metadata come from. + pub domain: MultiModelDomain, + /// Optional overrides to the domain. + #[serde(default)] + pub overrides: Option, + /// The networks of the model, each with the transfers into it. + pub networks: Vec, +} + +/// Where a multi-model takes its time domain, scenarios and metadata from. +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum MultiModelDomain { + /// A local model definition's base model, with that definition's overrides applied. + Model(ModelDomain), + /// A ModelSchema JSON file, whose network is ignored. + BaseModel(BaseModelDomain), + /// A time domain and scenarios given inline. + Inline(InlineDomain), +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelDomain { + pub model: String, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct BaseModelDomain { + /// Path (relative to root) to the ModelSchema JSON. + pub base_model: String, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct InlineDomain { + pub time: TimeDomain, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scenarios: Option, +} + +/// One network of a multi-model. +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct MultiModelNetwork { + pub name: String, + /// The definition the network starts from. Without one, the network holds only `include`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network: Option, + /// Per-network-set selections, included after the network's source. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include: Vec, + /// Transfers into this network from other networks of the multi-model. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub transfers: Vec, + /// Path (relative to root) to a JSON list of further transfers into this network. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transfers_file: Option, +} + +/// The definition a multi-model network starts from. +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +pub enum MultiModelNetworkSource { + /// A network definition. + Network(NetworkSource), + /// The network part of a model definition: its base model's network and everything merged + /// into it. The model's time domain, scenarios and overrides are not used. + Model(ModelSource), +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct NetworkSource { + pub network: String, +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ModelSource { + pub model: String, +} + +/// A multi-model definition resolved into its domain and networks. +struct ResolvedMultiModel { + domain: Option, + overrides: Vec, + networks: Vec, +} + +impl MultiModelDefinition { + /// Resolve this definition's domain and networks. /// /// Problems are appended in the order the definition is composed, so the first one is the /// error that composition reports. fn resolve<'a>( &'a self, - root: &Path, - project_base_model: &ProjectBaseModel, - networks: &mut NetworkExpander<'a>, + resolver: &mut Resolver<'a>, problems: &mut Vec, - ) -> ResolvedModel { - let base_model = match (&self.base_model, project_base_model) { - (Some(base_model), _) => match resolve_base_model(root, base_model) { - Ok(base_model) => Some(base_model), - Err(error) => { - problems.push(error.into()); + ) -> ResolvedMultiModel { + let mut overrides = Vec::new(); + let domain = match &self.domain { + MultiModelDomain::Model(ModelDomain { model }) => match resolver.manifest.find_model(model) { + Some(definition) => { + overrides.extend(definition.overrides.clone()); + resolver + .model_base_model(definition, problems) + .map(|(path, _)| ComposedDomain::BaseModel(path)) + } + None => { + problems.push(ProjectManifestProblem::ModelNotFound { + definition: self.name.clone(), + model: model.clone(), + }); None } }, - (None, ProjectBaseModel::Resolved(base_model)) => Some(base_model.clone()), - (None, ProjectBaseModel::Unresolved) => None, - (None, ProjectBaseModel::NotGiven) => { - problems.push(ProjectManifestProblem::BaseModelMissing { - definition: self.name.clone(), - }); - None + MultiModelDomain::BaseModel(BaseModelDomain { base_model }) => { + match resolve_base_model(resolver.root, base_model) { + Ok((path, _)) => Some(ComposedDomain::BaseModel(path)), + Err(error) => { + problems.push(error.into()); + None + } + } } + MultiModelDomain::Inline(InlineDomain { time, scenarios }) => Some(ComposedDomain::Inline { + time: time.clone(), + scenarios: scenarios.clone(), + }), }; + overrides.extend(self.overrides.clone()); + + add_duplicate_problems( + &self.networks, + |network| &network.name, + |name, count| ProjectManifestProblem::DuplicateMultiModelNetwork { + definition: self.name.clone(), + network: name, + count, + }, + problems, + ); + let networks = self + .networks + .iter() + .map(|network| network.resolve(self, resolver, problems)) + .collect(); - let project = networks.project; + ResolvedMultiModel { + domain, + overrides, + networks, + } + } +} + +impl MultiModelNetwork { + fn resolve<'a>( + &'a self, + multi_model: &MultiModelDefinition, + resolver: &mut Resolver<'a>, + problems: &mut Vec, + ) -> ComposedMultiModelNetwork { + // Problems in this network are reported against `/`. + let label = format!("{}/{}", multi_model.name, self.name); + let mut base_model = None; let mut expansion = Expansion::default(); - for name in &self.sub_networks { - if let Some(sub_network) = project.referred_sub_network(&self.name, name, problems) { - expansion.active_sub_networks.push(&sub_network.name); + match &self.network { + Some(MultiModelNetworkSource::Network(NetworkSource { network })) => { + if resolver.manifest.find_network(network).is_none() { + problems.push(ProjectManifestProblem::NetworkNotFound { + definition: label.clone(), + network: network.clone(), + }); + } + expansion.extend(resolver.expand(network, problems)); } + Some(MultiModelNetworkSource::Model(ModelSource { model })) => match resolver.manifest.find_model(model) { + Some(definition) => { + base_model = resolver.model_base_model(definition, problems); + expansion.extend(resolver.expand_model(definition, problems)); + } + None => problems.push(ProjectManifestProblem::ModelNotFound { + definition: label.clone(), + model: model.clone(), + }), + }, + None => {} } - // The base networks of the sub-networks activated here come straight after the base model. - for sub_network in &project.sub_networks { - if let Some(file) = &sub_network.base_network - && expansion.active_sub_networks.contains(&sub_network.name.as_str()) - { - expansion.files.push(SelectedFile { - file, - set: None, - position_offset: None, - network: None, - }); - } + expansion.files.extend(resolve_includes( + &label, + None, + &self.include, + resolver.project, + problems, + )); + expansion.dedup(); + check_composition(&label, &expansion, resolver.project, problems); + + let transfers = self.resolve_transfers(multi_model, resolver.root, problems); + let base_model = base_model.map(|(path, file)| { + let source = ProvenanceSource { + file, + network_set: None, + sub_network: None, + network: None, + }; + (path, source) + }); + ComposedMultiModelNetwork { + network: ComposedNetwork::new( + self.name.clone(), + base_model, + expansion.files.into_iter().map(SelectedFile::into_composed).collect(), + ), + transfers, } - for network in &self.networks { - if networks.manifest.find_network(network).is_none() { - problems.push(ProjectManifestProblem::NetworkNotFound { - definition: self.name.clone(), - network: network.clone(), - }); + } + + /// The transfers into this network, given inline and in `transfers_file`, checked against + /// the other networks of `multi_model`. + fn resolve_transfers( + &self, + multi_model: &MultiModelDefinition, + root: &Path, + problems: &mut Vec, + ) -> Vec { + let mut transfers = self.transfers.clone(); + if let Some(transfers_file) = &self.transfers_file { + match read_transfers(root, transfers_file) { + Ok(from_file) => transfers.extend(from_file), + Err(error) => problems.push(error.into()), } - expansion.extend(networks.expand(network, problems)); } - expansion - .files - .extend(resolve_includes(&self.name, None, &self.include, project, problems)); - expansion.dedup(); - check_active(&self.name, &expansion, problems); - // Set constraints apply to everything the model selects from a set, by any route. A - // sub-network's sets are only constrained when the sub-network is active. - for set in &project.sets { - if let Some(sub_network) = &set.sub_network - && !expansion.active_sub_networks.contains(&sub_network.as_str()) + let network = format!("{}/{}", multi_model.name, self.name); + for transfer in &transfers { + if transfer.from_network == self.name { + problems.push(ProjectManifestProblem::TransferFromSameNetwork { + network: network.clone(), + transfer: transfer.name.clone(), + }); + } else if !multi_model + .networks + .iter() + .any(|other| other.name == transfer.from_network) { - continue; + problems.push(ProjectManifestProblem::TransferSourceNotFound { + network: network.clone(), + transfer: transfer.name.clone(), + from_network: transfer.from_network.clone(), + }); } - let count = expansion - .files - .iter() - .filter(|selected| selected.set.is_some_and(|selected_set| std::ptr::eq(selected_set, set))) - .count(); - validate_constraints(&self.name, set, count, problems); } + add_duplicate_problems( + &transfers, + |transfer| &transfer.name, + |name, count| ProjectManifestProblem::DuplicateTransfer { + network: network.clone(), + transfer: name, + count, + }, + problems, + ); + transfers + } +} - ResolvedModel { - base_model, - includes: expansion.files.into_iter().map(SelectedFile::into_composed).collect(), - } +/// Read a JSON list of transfers from `transfers_file`, relative to `root`. +fn read_transfers(root: &Path, transfers_file: &str) -> Result, ManifestResolutionError> { + let candidate = strict_relative_path("transfers file", transfers_file)?.to_path(root); + if !candidate.exists() { + return Err(ManifestResolutionError::TransfersFileNotFound { path: candidate }); } + let path = canonicalize_contained(root, &candidate, "transfers file")?; + let data = std::fs::read_to_string(&path).map_err(|source| ManifestResolutionError::TransfersFileRead { + path: candidate.clone(), + source, + })?; + serde_json::from_str(&data).map_err(|source| ManifestResolutionError::TransfersFileJson { + path: candidate, + source, + }) } #[derive(Clone, Serialize, Deserialize, JsonSchema)] @@ -1414,6 +1859,7 @@ mod test { fn manifest(base_model: &str, network_sets: Vec, include: Vec) -> ProjectManifest { ProjectManifest { + multi_models: vec![], base_model: Some(base_model.to_string()), sub_networks: vec![], network_sets, @@ -1851,6 +2297,7 @@ mod test { fn network_definitions_compose_in_extension_order_and_record_provenance() { let root = project_with_nets(); let manifest = ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![], network_sets: vec![set("nets", None, None, None)], @@ -1905,6 +2352,7 @@ mod test { fn files_reached_by_several_routes_are_included_once() { let root = project_with_nets(); let manifest = ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![], network_sets: vec![set("nets", None, None, None)], @@ -1934,6 +2382,7 @@ mod test { fn set_constraints_apply_to_the_whole_model_not_each_network() { let root = project_with_nets(); let manifest = ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![], network_sets: vec![set("nets", None, None, Some(1))], @@ -1958,6 +2407,7 @@ mod test { fn network_cycles_are_reported_once() { let root = project_with_nets(); let manifest = ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![], network_sets: vec![set("nets", None, None, None)], @@ -1986,6 +2436,7 @@ mod test { fn missing_network_references_are_reported_where_they_are_made() { let root = project_with_nets(); let manifest = ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![], network_sets: vec![set("nets", None, None, None)], @@ -2014,6 +2465,7 @@ mod test { fn network_problems_are_reported_once_however_many_models_use_them() { let root = project_with_nets(); let manifest = ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![], network_sets: vec![set("nets", None, None, None)], @@ -2040,6 +2492,7 @@ mod test { let mut with_base = model("with-base", vec![]); with_base.base_model = Some("other.json".to_string()); let manifest = ProjectManifest { + multi_models: vec![], base_model: None, sub_networks: vec![], network_sets: vec![set("nets", None, None, None)], @@ -2124,6 +2577,7 @@ mod test { fn sub_network_manifest(models: Vec, networks: Vec) -> ProjectManifest { ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![ sub_network("upper", Some("base.json"), vec![set("options", None, None, None)]), @@ -2349,6 +2803,289 @@ mod test { )); } + fn transfer(from_network: &str, node: &str, name: &str) -> MultiNetworkTransfer { + serde_json::from_value(serde_json::json!({ + "from_network": from_network, + "metric": {"type": "Node", "name": node}, + "name": name, + })) + .unwrap() + } + + fn multi_network(name: &str, network: Option) -> MultiModelNetwork { + MultiModelNetwork { + name: name.to_string(), + network, + include: vec![], + transfers: vec![], + transfers_file: None, + } + } + + fn from_model(model: &str) -> Option { + Some(MultiModelNetworkSource::Model(ModelSource { + model: model.to_string(), + })) + } + + fn from_network(network: &str) -> Option { + Some(MultiModelNetworkSource::Network(NetworkSource { + network: network.to_string(), + })) + } + + /// A manifest over `project_with_nets` with a model `company` (base model plus `a.json`), a + /// network definition `river` (`b.json`) and a multi-model `joint` that links them. + fn multi_model_manifest(networks: Vec) -> ProjectManifest { + ProjectManifest { + multi_models: vec![MultiModelDefinition { + name: "joint".to_string(), + domain: MultiModelDomain::Model(ModelDomain { + model: "company".to_string(), + }), + overrides: None, + networks, + }], + base_model: Some("base.json".to_string()), + sub_networks: vec![], + network_sets: vec![set("nets", None, None, None)], + networks: vec![network( + "river", + vec![], + vec![selection("nets", Some(vec!["b.json"]), false)], + )], + models: vec![model("company", vec![selection("nets", Some(vec!["a.json"]), false)])], + } + } + + fn joint_networks() -> Vec { + let mut river = multi_network("river", from_network("river")); + river.include = vec![selection("nets", Some(vec!["c.json"]), false)]; + river.transfers = vec![transfer("company", "a", "company-a")]; + vec![multi_network("company", from_model("company")), river] + } + + #[test] + fn multi_model_networks_come_from_models_and_network_definitions() { + let root = project_with_nets(); + let manifest = multi_model_manifest(joint_networks()); + assert!(manifest.validate(root.path()).is_ok()); + + let composed = manifest.compose_multi_model(root.path(), "joint").unwrap(); + let paths: Vec<_> = composed + .networks() + .iter() + .map(|network| network.network.all_paths()) + .collect(); + assert_eq!( + paths, + [ + net_paths(root.path(), &["base.json", "nets/a.json"]), + net_paths(root.path(), &["nets/b.json", "nets/c.json"]), + ] + ); + + let schema = composed + .load() + .unwrap() + .into_multi_model_schema(&Default::default()) + .unwrap(); + assert_eq!(schema.metadata.title, "joint"); + let names: Vec<_> = schema.networks.iter().map(|entry| entry.name.as_str()).collect(); + assert_eq!(names, ["company", "river"]); + let pywr_schema::NetworkSchemaRef::Inline(river) = &schema.networks[1].network else { + panic!("expected an inline network"); + }; + assert_eq!(river.nodes.len(), 2); + assert_eq!(schema.networks[1].transfers[0].name, "company-a"); + + assert!(matches!( + manifest.compose(root.path(), "joint"), + Ok(crate::Composed::MultiModel(_)) + )); + assert!(matches!( + manifest.compose(root.path(), "company"), + Ok(crate::Composed::Model(_)) + )); + } + + #[test] + fn multi_model_domain_overrides_apply_in_order() { + let root = project_with_nets(); + let time = |end: &str| -> pywr_schema::model::TimeDomain { + serde_json::from_value(serde_json::json!({ + "start": "2020-01-01T00:00:00", "end": end, "timestep": {"type": "Days", "days": 1} + })) + .unwrap() + }; + let overrides = |end: &str| DefinitionOverrides { + time: Some(time(end)), + scenarios: None, + }; + let mut manifest = multi_model_manifest(joint_networks()); + manifest.models[0].overrides = Some(overrides("2020-02-01T00:00:00")); + let domain_end = |manifest: &ProjectManifest| { + let schema = manifest + .compose_multi_model(root.path(), "joint") + .unwrap() + .load() + .unwrap() + .into_multi_model_schema(&Default::default()) + .unwrap(); + serde_json::to_value(schema.time).unwrap()["end"].clone() + }; + assert_eq!(domain_end(&manifest), "2020-02-01T00:00:00"); + + manifest.multi_models[0].overrides = Some(overrides("2020-03-01T00:00:00")); + assert_eq!(domain_end(&manifest), "2020-03-01T00:00:00"); + + manifest.multi_models[0].domain = MultiModelDomain::Inline(InlineDomain { + time: time("2020-04-01T00:00:00"), + scenarios: None, + }); + manifest.multi_models[0].overrides = None; + assert_eq!(domain_end(&manifest), "2020-04-01T00:00:00"); + } + + #[test] + fn transfers_must_come_from_another_network_and_have_unique_names() { + let root = project_with_nets(); + let mut networks = joint_networks(); + networks[1].transfers = vec![ + transfer("river", "b", "own"), + transfer("nowhere", "a", "lost"), + transfer("company", "a", "twice"), + transfer("company", "a", "twice"), + ]; + let manifest = multi_model_manifest(networks); + let report = manifest.validate(root.path()).unwrap_err(); + let messages: Vec<_> = report.problems.iter().map(ToString::to_string).collect(); + assert_eq!( + messages, + [ + "The transfer `own` into `joint/river` is from the same network.", + "The transfer `lost` into `joint/river` is from the network `nowhere`, which does not exist.", + "The name `twice` is used by 2 transfers into `joint/river`, but each name must be unique.", + ] + ); + } + + #[test] + fn transfers_are_read_from_a_transfers_file() { + let root = project_with_nets(); + write_json( + &root.path().join("links/transfers.json"), + serde_json::json!([{"from_network": "company", "metric": {"type": "Node", "name": "a"}, "name": "from-file"}]), + ); + let mut networks = joint_networks(); + networks[1].transfers_file = Some("links/transfers.json".to_string()); + let manifest = multi_model_manifest(networks); + let composed = manifest.compose_multi_model(root.path(), "joint").unwrap(); + let names: Vec<_> = composed.networks()[1] + .transfers + .iter() + .map(|transfer| transfer.name.as_str()) + .collect(); + assert_eq!(names, ["company-a", "from-file"]); + + let mut networks = joint_networks(); + networks[1].transfers_file = Some("links/missing.json".to_string()); + assert!(matches!( + multi_model_manifest(networks).compose_multi_model(root.path(), "joint"), + Err(ComposeModelError::Resolution( + ManifestResolutionError::TransfersFileNotFound { .. } + )) + )); + } + + #[test] + fn transfers_must_read_components_of_their_source_network() { + let root = project_with_nets(); + let mut networks = joint_networks(); + networks[1].transfers = vec![transfer("company", "missing", "company-missing")]; + let Err(error) = multi_model_manifest(networks) + .compose_multi_model(root.path(), "joint") + .unwrap() + .load() + .unwrap() + .into_multi_model_schema(&Default::default()) + else { + panic!("expected a missing transfer component"); + }; + assert_eq!( + error.to_string(), + "The transfer `company-missing` into `river` reads the node `missing` from `company`, which does not exist there." + ); + } + + #[test] + fn multi_model_problems_are_attributed_to_the_network() { + let root = project_with_nets(); + let mut networks = joint_networks(); + networks[0].network = from_model("absent"); + networks[1].include = vec![selection("nets", Some(vec!["missing.json"]), false)]; + networks.push(multi_network("company", None)); + let report = multi_model_manifest(networks).validate(root.path()).unwrap_err(); + let messages: Vec<_> = report.problems.iter().map(ToString::to_string).collect(); + assert_eq!( + messages, + [ + "The multi-model `joint` has 2 networks named `company`, but each name must be unique.", + "The definition `joint/company` refers to the model definition `absent`, which does not exist.", + "The definition `joint/river` selects the file `missing.json`, which was not found in network set `nets`.", + ] + ); + } + + #[test] + fn models_and_multi_models_share_names() { + let root = project_with_nets(); + let mut manifest = multi_model_manifest(joint_networks()); + manifest.multi_models[0].name = "company".to_string(); + assert!(matches!( + manifest.validate(root.path()).unwrap_err().problems.as_slice(), + [ProjectManifestProblem::DuplicateDefinition { definition, count: 2 }, ..] if definition == "company" + )); + } + + #[test] + fn multi_model_json_accepts_each_domain_and_network_source() { + let manifest: super::super::ProjectManifest = serde_json::from_str( + r#"{ + "format": "v1", + "multi_models": [ + { + "name": "from-model", + "domain": { "model": "full" }, + "networks": [ + { "name": "a", "network": { "model": "full" } }, + { "name": "b", "network": { "network": "upgraded" }, "transfers_file": "links.json" } + ] + }, + { "name": "from-file", "domain": { "base_model": "base.json" }, "networks": [] }, + { + "name": "inline", + "domain": { "time": { "start": "2020-01-01T00:00:00", "end": "2020-12-31T00:00:00", "timestep": { "type": "Days", "days": 1 } } }, + "networks": [{ "name": "c", "include": [{ "set": "nets", "include_all": true }] }] + } + ] + }"#, + ) + .unwrap(); + let super::super::ProjectManifest::V1(manifest) = manifest; + let [from_model, from_file, inline] = manifest.multi_models.as_slice() else { + panic!("expected three multi-models"); + }; + assert!(matches!(from_model.domain, MultiModelDomain::Model(_))); + assert!(matches!( + from_model.networks[1].network, + Some(MultiModelNetworkSource::Network(_)) + )); + assert!(matches!(from_file.domain, MultiModelDomain::BaseModel(_))); + assert!(matches!(inline.domain, MultiModelDomain::Inline(_))); + assert!(inline.networks[0].network.is_none()); + } + #[test] fn omitted_set_constraints_are_reported_in_declaration_order() { let root = tempdir().unwrap(); @@ -2517,6 +3254,7 @@ mod test { touch(&root.path().join("base.json")); std::fs::create_dir(root.path().join("nets")).unwrap(); let manifest = ProjectManifest { + multi_models: vec![], base_model: Some("base.json".to_string()), sub_networks: vec![], network_sets: vec![set("nets", None, Some(2), Some(1)), set("nets", None, None, None)], diff --git a/pywr-project/src/project.rs b/pywr-project/src/project.rs index 918c3645..90684721 100644 --- a/pywr-project/src/project.rs +++ b/pywr-project/src/project.rs @@ -1,6 +1,6 @@ use crate::error::ComposeModelError; use crate::manifest::ProjectManifest; -use crate::{ComposedModel, ComposedNetwork, ProjectError}; +use crate::{Composed, ComposedModel, ComposedMultiModel, ComposedNetwork, ProjectError}; use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -67,9 +67,19 @@ impl Project { /// Compose a model from the project manifest and a definition name. pub fn compose_model(&self, definition_name: &str) -> Result { + self.manifest.compose_model(&self.root, definition_name) + } + + /// Compose a model or multi-model from the project manifest and a definition name. + pub fn compose(&self, definition_name: &str) -> Result { self.manifest.compose(&self.root, definition_name) } + /// Compose a multi-network model from the project manifest and a multi-model definition name. + pub fn compose_multi_model(&self, definition_name: &str) -> Result { + self.manifest.compose_multi_model(&self.root, definition_name) + } + /// Compose a network from the project manifest and a network definition name. pub fn compose_network(&self, network_name: &str) -> Result { self.manifest.compose_network(&self.root, network_name) @@ -88,6 +98,7 @@ mod tests { let manifest = ProjectManifest::V1(v1::ProjectManifest { base_model: Some("base_model.json".to_string()), sub_networks: vec![], + multi_models: vec![], network_sets: vec![], networks: vec![], models: vec![], diff --git a/pywr-project/tests/example.rs b/pywr-project/tests/example.rs index aa41a3a3..403a8772 100644 --- a/pywr-project/tests/example.rs +++ b/pywr-project/tests/example.rs @@ -21,4 +21,13 @@ fn example_project_validates_and_every_model_composes() { .into_model_schema(&Default::default()) .unwrap(); } + for multi_model in &manifest.multi_models { + project + .compose_multi_model(&multi_model.name) + .unwrap() + .load() + .unwrap() + .into_multi_model_schema(&Default::default()) + .unwrap(); + } }