diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index dc7e60aea..1bb17e844 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -55,12 +55,16 @@ strategy: IMAGE_NAME: ubuntu-latest PYTHON_VERSION: 3.9 CODECOV: True # Only run on one build - macos-py3.9: + linux-py3.11: + IMAGE_NAME: ubuntu-latest + PYTHON_VERSION: 3.11 + CODECOV: True # Only run on one build + macos-py3.11: IMAGE_NAME: macOS-latest - PYTHON_VERSION: 3.9 - windows-py3.9: + PYTHON_VERSION: 3.11 + windows-py3.11: IMAGE_NAME: windows-latest - PYTHON_VERSION: 3.9 + PYTHON_VERSION: 3.11 steps: - bash: echo "##vso[task.prependpath]$CONDA/bin" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 208ccf59e..0f5f0fce4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,25 @@ +default_language_version: + python: python3 + repos: - repo: https://github.com/psf/black - rev: stable + rev: 23.10.0 hooks: - id: black - - repo: https://github.com/pycqa/isort - rev: 5.11.2 + - repo: https://github.com/astral-sh/ruff-pre-commit # https://beta.ruff.rs/docs/usage/#github-action + rev: v0.1.1 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + + - repo: https://github.com/nbQA-dev/nbQA + rev: 1.7.0 hooks: - - id: isort - name: isort (python) + - id: nbqa-black + - id: nbqa-ruff + args: [--fix, --exit-non-zero-on-fix] + +ci: # https://pre-commit.ci/ + autofix_prs: false + autoupdate_schedule: monthly \ No newline at end of file diff --git a/calliope/backend/backends.py b/calliope/backend/backends.py index 2184323f3..bb6f558e5 100644 --- a/calliope/backend/backends.py +++ b/calliope/backend/backends.py @@ -1076,17 +1076,20 @@ def _to_pyomo_param( If both `val` and `default` are np.nan/None, return np.nan. Otherwise return ObjParameter(val/default). """ - with pd.option_context("mode.use_inf_as_na", use_inf_as_na): - if pd.isnull(val): - if pd.isnull(default): - param = np.nan - else: - param = ObjParameter(default) - self._instance.parameters[name].append(param) + if use_inf_as_na: + val = np.nan if val in [np.inf, -np.inf] else val + default = np.nan if default in [np.inf, -np.inf] else default + + if pd.isnull(val): + if pd.isnull(default): + param = np.nan else: - param = ObjParameter(val) + param = ObjParameter(default) self._instance.parameters[name].append(param) - return param + else: + param = ObjParameter(val) + self._instance.parameters[name].append(param) + return param def _to_pyomo_constraint( self, diff --git a/calliope/backend/helper_functions.py b/calliope/backend/helper_functions.py index 6b96c2067..877462083 100644 --- a/calliope/backend/helper_functions.py +++ b/calliope/backend/helper_functions.py @@ -7,11 +7,12 @@ Functions that can be used to process data in math `where` and `expression` strings. """ +import functools import re from abc import ABC, abstractmethod from typing import Any, Literal, Mapping, Union, overload -import pandas as pd +import numpy as np import xarray as xr from calliope.exceptions import BackendError @@ -189,12 +190,11 @@ def as_array(self, parameter: str, *, over: Union[str, list[str]]) -> xr.DataArr """ if parameter in self._kwargs["model_data"].data_vars: parameter_da = self._kwargs["model_data"][parameter] - with pd.option_context("mode.use_inf_as_na", True): - bool_parameter_da = ( - parameter_da.where(pd.notnull(parameter_da)) # type: ignore - .notnull() - .any(dim=over, keep_attrs=True) - ) + bool_parameter_da = ( + parameter_da.notnull() + & (parameter_da != np.inf) + & (parameter_da != -np.inf) + ).any(dim=over, keep_attrs=True) else: bool_parameter_da = xr.DataArray(False) return bool_parameter_da @@ -227,7 +227,8 @@ def as_array( Returns: xr.DataArray: Array with dimensions reduced by applying a summation over the dimensions given in `over`. - NaNs are ignored (xarray.DataArray.sum arg: `skipna: True`) and if all values along the dimension(s) are NaN, the summation will lead to a NaN (xarray.DataArray.sum arg: `min_count=1`). + NaNs are ignored (xarray.DataArray.sum arg: `skipna: True`) and if all values along the dimension(s) are NaN, + the summation will lead to a NaN (xarray.DataArray.sum arg: `min_count=1`). """ return array.sum(over, min_count=1, skipna=True) @@ -282,7 +283,8 @@ def as_array( self, array: xr.DataArray, carrier_tier: Literal["in", "out"] ) -> xr.DataArray: """Reduce expression array data by selecting the carrier that corresponds to the primary carrier and then dropping the `carriers` dimension. - This function is only valid for `conversion_plus` technologies, so should only be included in a math component if the `where` string includes `inheritance(conversion_plus)` or an equivalent expression. + This function is only valid for `conversion_plus` technologies, + so should only be included in a math component if the `where` string includes `inheritance(conversion_plus)` or an equivalent expression. Args: array (xr.DataArray): Expression array. @@ -353,6 +355,9 @@ def as_array( The lookup array assigns the value at "B" to "A" and vice versa. "C" is masked since the lookup array value is NaN. """ + # Inspired by https://github.com/pydata/xarray/issues/1553#issuecomment-748491929 + # Reindex does not presently support vectorized lookups: https://github.com/pydata/xarray/issues/1553 + # Sel does (e.g. https://github.com/pydata/xarray/issues/4630) but can't handle missing keys dims = set(lookup_arrays.keys()) missing_dims_in_component = dims.difference(array.dims) @@ -368,24 +373,30 @@ def as_array( f"All lookup arrays used to select items from `{array.name}` must be indexed over the dimensions {dims}" ) - stacked_and_dense_lookup_arrays = { - # Although we have the lookup array, its values are backend objects, - # so we grab the same array from the unadulterated model data. - # FIXME: do not add lookup tables as backend objects. - dim_name: self._kwargs["model_data"][lookup.name] - # Stacking ensures that the dimensions on `component` are not reordered on calling `.sel()`. - .stack(idx=list(dims)) - # Cannot select on NaNs, so we drop them all. - .dropna("idx") - for dim_name, lookup in lookup_arrays.items() - } - sliced_component = array.sel(stacked_and_dense_lookup_arrays) + dim = "dim_0" + ixs = {} + masks = [] + + # Turn string lookup values to numeric ones. + # We stack the dimensions to handle multidimensional lookups + for index_dim, index in lookup_arrays.items(): + stacked_lookup = self._kwargs["model_data"][index.name].stack({dim: dims}) + ix = array.indexes[index_dim].get_indexer(stacked_lookup) + ixs[index_dim] = xr.DataArray( + np.fmax(0, ix), + coords={dim: stacked_lookup[dim]}, + ) + masks.append(ix >= 0) - return ( - sliced_component.drop_vars(dims) - .unstack("idx") - .reindex_like(array, copy=False) - ) + # Create a mask to nullify any lookup values that are not given (i.e., are np.nan in the lookup array) + mask = functools.reduce(lambda x, y: x & y, masks) + + result = array[ixs] + + if not mask.all(): + result[{dim: ~mask}] = np.nan + unstacked_result = result.drop_vars(dims).unstack(dim) + return unstacked_result class GetValAtIndex(ParsingHelperFunction): diff --git a/calliope/backend/latex_backend.py b/calliope/backend/latex_backend.py index 35c5d4219..89fbc2331 100644 --- a/calliope/backend/latex_backend.py +++ b/calliope/backend/latex_backend.py @@ -67,7 +67,9 @@ def write( # noqa: F811 If given, will write the built mathematical formulation to a file with the given extension as the file format. Defaults to None. format (Optional["tex", "rst", "md"], optional): - Not required if filename is given (as the format will be automatically inferred). Required if expecting a string return from calling this function. The LaTeX math will be embedded in a document of the given format (tex=LaTeX, rst=reStructuredText, md=Markdown). Defaults to None. + Not required if filename is given (as the format will be automatically inferred). + Required if expecting a string return from calling this function. The LaTeX math will be embedded in a document of the given format (tex=LaTeX, rst=reStructuredText, md=Markdown). + Defaults to None. Raises: exceptions.ModelError: Math strings need to be built first (`build`) diff --git a/calliope/backend/parsing.py b/calliope/backend/parsing.py index 564aa6a79..c49f27d3e 100644 --- a/calliope/backend/parsing.py +++ b/calliope/backend/parsing.py @@ -616,7 +616,8 @@ def extend_equation_list_with_expression_group( Returns: list[ParsedBackendEquation]: - Expanded list of parsed equations with the product of all references to items from the `expression_group` producing a new equation object. E.g., if the input equation object has a reference to an slice which itself has two expression options, two equation objects will be added to the return list. + Expanded list of parsed equations with the product of all references to items from the `expression_group` producing a new equation object. + E.g., if the input equation object has a reference to an slice which itself has two expression options, two equation objects will be added to the return list. """ if expression_group == "sub_expressions": equation_items = parsed_equation.find_sub_expressions() @@ -643,11 +644,10 @@ def extend_equation_list_with_expression_group( ] def combine_exists_and_foreach(self, model_data: xr.Dataset) -> xr.DataArray: - """ - Generate a multi-dimensional boolean array based on the sets - over which the constraint is to be built (defined by "foreach") and the - model `exists` array. - The `exists` array is a boolean array defining the structure of the model and is True for valid combinations of technologies consuming/producing specific carriers at specific nodes. It is indexed over ["nodes", "techs", "carriers", "carrier_tiers"]. + """Generate a multi-dimensional boolean array based on the sets over which the constraint is to be built (defined by "foreach") and the model `exists` array. + + The `exists` array is a boolean array defining the structure of the model and is True for valid combinations of technologies consuming/producing specific carriers at specific nodes. + It is indexed over ["nodes", "techs", "carriers", "carrier_tiers"]. Args: model_data (xr.Dataset): Calliope model dataset. @@ -682,7 +682,8 @@ def generate_top_level_where_array( Args: model_data (xr.Dataset): Calliope model input data. align_to_foreach_sets (bool, optional): - By default, all foreach arrays have the dimensions ("nodes", "techs", "carriers", "carrier_tiers") as well as any additional dimensions provided by the component's "foreach" key. If this argument is True, the dimensions not included in "foreach" are removed from the array. + By default, all foreach arrays have the dimensions ("nodes", "techs", "carriers", "carrier_tiers") as well as any additional dimensions provided by the component's "foreach" key. + If this argument is True, the dimensions not included in "foreach" are removed from the array. Defaults to True. break_early (bool, optional): If any intermediate array has no valid elements (i.e. all are False), the function will return that array rather than continuing - this saves time and memory on large models. diff --git a/calliope/backend/where_parser.py b/calliope/backend/where_parser.py index a66b23b98..1d937b7f2 100644 --- a/calliope/backend/where_parser.py +++ b/calliope/backend/where_parser.py @@ -7,7 +7,6 @@ from typing import Any, Union import numpy as np -import pandas as pd import pyparsing as pp import xarray as xr @@ -179,9 +178,8 @@ def as_latex(self, model_data: xr.Dataset, apply_where: bool = True) -> str: def _data_var_exists(self, model_data: xr.Dataset) -> xr.DataArray: "mask by setting all (NaN | INF/-INF) to False, otherwise True" - model_data_var = model_data.get(self.data_var, xr.DataArray(None)) - with pd.option_context("mode.use_inf_as_na", True): - return model_data_var.where(pd.notnull(model_data_var)).notnull() # type: ignore + var = model_data.get(self.data_var, xr.DataArray(np.nan)) + return var.notnull() & (var != np.inf) & (var != -np.inf) def _data_var_with_default(self, model_data: xr.Dataset) -> xr.DataArray: "Access data var and fill with default values. Return default value as an array if var does not exist" diff --git a/calliope/config/defaults.yaml b/calliope/config/defaults.yaml index 69f336713..6cabaea1d 100644 --- a/calliope/config/defaults.yaml +++ b/calliope/config/defaults.yaml @@ -44,7 +44,7 @@ model: time: {} # Optional settings to adjust time resolution, see :ref:`time_clustering` for the available options timeseries_data_path: null # Path to time series data timeseries_data: null # Dict of dataframes with time series data (when passing in dicts rather than YAML files to Model constructor) - timeseries_dateformat: "%Y-%m-%d %H:%M:%S" # Timestamp format of all time series data when read from file + timeseries_dateformat: "ISO8601" # Timestamp format of all time series data when read from file. "ISO8601" means "YYYY-mm-dd HH:MM:SS". file_allowed: [ "clustering_func", "energy_eff", diff --git a/calliope/core/io.py b/calliope/core/io.py index 72756a440..b4a18c373 100644 --- a/calliope/core/io.py +++ b/calliope/core/io.py @@ -11,6 +11,9 @@ import os +# We import netCDF4 before xarray to mitigate a numpy warning: +# https://github.com/pydata/xarray/issues/7259 +import netCDF4 # noqa: F401 import numpy as np import pandas as pd import xarray as xr @@ -27,7 +30,7 @@ def read_netcdf(path): calliope_version = model_data.attrs.get("calliope_version", False) if calliope_version: - if not str(calliope_version) in __version__: + if str(calliope_version) not in __version__: exceptions.warn( "This model data was created with Calliope version {}, " "but you are running {}. Proceed with caution!".format( @@ -43,6 +46,7 @@ def read_netcdf(path): # Convert empty strings back to np.NaN # TODO: revert when this issue is solved: https://github.com/pydata/xarray/issues/1647 + # which it might be once this is merged: https://github.com/pydata/xarray/pull/7869 for var_name, var_array in model_data.data_vars.items(): if var_array.dtype.kind in ["U", "O"]: model_data[var_name] = var_array.where(lambda x: x != "") diff --git a/calliope/core/model.py b/calliope/core/model.py index bb47a2736..49a40be48 100644 --- a/calliope/core/model.py +++ b/calliope/core/model.py @@ -589,7 +589,9 @@ def info(self) -> str: return "\n".join(info_strings) def validate_math_strings(self, math_dict: dict) -> None: - """Validate that `expression` and `where` strings of a dictionary containing string mathematical formulations can be successfully parsed. This function can be used to test custom math before attempting to build the optimisation problem. + """Validate that `expression` and `where` strings of a dictionary containing string mathematical formulations can be successfully parsed. + + This function can be used to test custom math before attempting to build the optimisation problem. NOTE: strings are not checked for evaluation validity. Evaluation issues will be raised only on calling `Model.build()`. diff --git a/calliope/core/util/generate_runs.py b/calliope/core/util/generate_runs.py index 51a5ae513..6196de98f 100644 --- a/calliope/core/util/generate_runs.py +++ b/calliope/core/util/generate_runs.py @@ -166,8 +166,10 @@ def generate_sbatch_script( ) if ":" not in cluster_time: - # Assuming time given as minutes, so needs changing to %H:%M%S - cluster_time = pd.to_datetime(cluster_time, unit="m").strftime("%H:%M:%S") + # Assuming time given as minutes, so needs changing to %H:%M:%S + cluster_time = pd.to_datetime(float(cluster_time), unit="m").strftime( + "%H:%M:%S" + ) lines = [ "#!/bin/bash", diff --git a/calliope/core/util/tools.py b/calliope/core/util/tools.py index 3d40fa12a..c8f861536 100644 --- a/calliope/core/util/tools.py +++ b/calliope/core/util/tools.py @@ -9,9 +9,8 @@ import sys from typing import Callable, TypeVar -from typing_extensions import ParamSpec - import jsonschema +from typing_extensions import ParamSpec from calliope.exceptions import print_warnings_and_raise_errors diff --git a/calliope/exceptions.py b/calliope/exceptions.py index 7d6a14a77..ffb34b252 100644 --- a/calliope/exceptions.py +++ b/calliope/exceptions.py @@ -93,7 +93,9 @@ def print_warnings_and_raise_errors( List of error strings or dictionary of error strings. If None or an empty list, no errors will be raised. Defaults to None. - during (str, optional): substring that will be placed at the top of the concated list of warnings/errors to point to during which phase of data processing they occured. Defaults to "model processing". + during (str, optional): + Substring that will be placed at the top of the concated list of warnings/errors to point to during which phase of data processing they occured. + Defaults to "model processing". bullet (str, optional): Type of bullet points to use. Defaults to " * ". Raises: diff --git a/calliope/preprocess/checks.py b/calliope/preprocess/checks.py index f5b500232..b906cb205 100644 --- a/calliope/preprocess/checks.py +++ b/calliope/preprocess/checks.py @@ -105,7 +105,7 @@ def check_initial(config_model): # Check for version mismatch model_version = config_model.model.get("calliope_version", False) if model_version: - if not str(model_version) in __version__: + if str(model_version) not in __version__: model_warnings.append( "Model configuration specifies calliope_version={}, " "but you are running {}. Proceed with caution!".format( diff --git a/calliope/preprocess/model_run.py b/calliope/preprocess/model_run.py index 686d3061c..253df942f 100644 --- a/calliope/preprocess/model_run.py +++ b/calliope/preprocess/model_run.py @@ -521,7 +521,7 @@ def load_timeseries_from_dataframe(timeseries_dataframes, tskey): def _parser(x, dtformat): - return pd.to_datetime(x, format=dtformat, exact=False) + return pd.to_datetime(x, format=dtformat) def _get_names(config): @@ -603,7 +603,7 @@ def process_timeseries_data(config_model, model_run, timeseries_dataframes): if subset_time_config is not None: # Test parsing dates first, to make sure they fit our required subset format try: - subset_time = _parser(subset_time_config, "%Y-%m-%d %H:%M:%S") + subset_time = _parser(subset_time_config, "ISO8601") except ValueError as e: raise exceptions.ModelError( "Timeseries subset must be in ISO format (anything up to the " diff --git a/calliope/test/common/lp_files/balance_conversion.lp b/calliope/test/common/lp_files/balance_conversion.lp index d10c3d276..55c804eab 100644 --- a/calliope/test/common/lp_files/balance_conversion.lp +++ b/calliope/test/common/lp_files/balance_conversion.lp @@ -2,40 +2,38 @@ min objectives(0): -+2 ONE_VAR_CONSTANT ++2.0 ONE_VAR_CONSTANT s.t. c_e_constraints(balance_conversion)(0)_: -+0.90000000000000002 variables(carrier_con)(a__test_conversion__gas__2005_01_01_00_00) +1 variables(carrier_prod)(a__test_conversion__heat__2005_01_01_00_00) -= 0 ++0.9 variables(carrier_con)(a__test_conversion__gas__2005_01_01_00_00) += 0.0 c_e_constraints(balance_conversion)(1)_: -+0.90000000000000002 variables(carrier_con)(a__test_conversion__gas__2005_01_01_01_00) +1 variables(carrier_prod)(a__test_conversion__heat__2005_01_01_01_00) -= 0 ++0.9 variables(carrier_con)(a__test_conversion__gas__2005_01_01_01_00) += 0.0 c_e_constraints(balance_conversion)(2)_: -+0.90000000000000002 variables(carrier_con)(b__test_conversion__gas__2005_01_01_00_00) +1 variables(carrier_prod)(b__test_conversion__heat__2005_01_01_00_00) -= 0 ++0.9 variables(carrier_con)(b__test_conversion__gas__2005_01_01_00_00) += 0.0 c_e_constraints(balance_conversion)(3)_: -+0.90000000000000002 variables(carrier_con)(b__test_conversion__gas__2005_01_01_01_00) +1 variables(carrier_prod)(b__test_conversion__heat__2005_01_01_01_00) -= 0 - -c_e_ONE_VAR_CONSTANT: -ONE_VAR_CONSTANT = 1.0 ++0.9 variables(carrier_con)(b__test_conversion__gas__2005_01_01_01_00) += 0.0 bounds + 1 <= ONE_VAR_CONSTANT <= 1 0 <= variables(carrier_prod)(a__test_conversion__heat__2005_01_01_00_00) <= +inf + -inf <= variables(carrier_con)(a__test_conversion__gas__2005_01_01_00_00) <= 0 0 <= variables(carrier_prod)(a__test_conversion__heat__2005_01_01_01_00) <= +inf + -inf <= variables(carrier_con)(a__test_conversion__gas__2005_01_01_01_00) <= 0 0 <= variables(carrier_prod)(b__test_conversion__heat__2005_01_01_00_00) <= +inf + -inf <= variables(carrier_con)(b__test_conversion__gas__2005_01_01_00_00) <= 0 0 <= variables(carrier_prod)(b__test_conversion__heat__2005_01_01_01_00) <= +inf - -inf <= variables(carrier_con)(a__test_conversion__gas__2005_01_01_00_00) <= 0 - -inf <= variables(carrier_con)(a__test_conversion__gas__2005_01_01_01_00) <= 0 - -inf <= variables(carrier_con)(b__test_conversion__gas__2005_01_01_00_00) <= 0 - -inf <= variables(carrier_con)(b__test_conversion__gas__2005_01_01_01_00) <= 0 + -inf <= variables(carrier_con)(b__test_conversion__gas__2005_01_01_01_00) <= 0 end diff --git a/calliope/test/common/lp_files/carrier_production_max.lp b/calliope/test/common/lp_files/carrier_production_max.lp index a268c19ea..b04ae1aa9 100644 --- a/calliope/test/common/lp_files/carrier_production_max.lp +++ b/calliope/test/common/lp_files/carrier_production_max.lp @@ -2,90 +2,88 @@ min objectives(0): -+2 ONE_VAR_CONSTANT ++2.0 ONE_VAR_CONSTANT s.t. c_u_constraints(carrier_production_max)(0)_: +1 variables(carrier_prod)(a__test_supply_elec__electricity__2005_01_01_00_00) --1 variables(energy_cap)(a__test_supply_elec) -<= 0 +-1.0 variables(energy_cap)(a__test_supply_elec) +<= 0.0 c_u_constraints(carrier_production_max)(1)_: +-1.0 variables(energy_cap)(a__test_supply_elec) +1 variables(carrier_prod)(a__test_supply_elec__electricity__2005_01_01_01_00) --1 variables(energy_cap)(a__test_supply_elec) -<= 0 +<= 0.0 c_u_constraints(carrier_production_max)(2)_: +1 variables(carrier_prod)(a__test_transmission_elec_b__electricity__2005_01_01_00_00) --1 variables(energy_cap)(a__test_transmission_elec_b) -<= 0 +-1.0 variables(energy_cap)(a__test_transmission_elec_b) +<= 0.0 c_u_constraints(carrier_production_max)(3)_: +-1.0 variables(energy_cap)(a__test_transmission_elec_b) +1 variables(carrier_prod)(a__test_transmission_elec_b__electricity__2005_01_01_01_00) --1 variables(energy_cap)(a__test_transmission_elec_b) -<= 0 +<= 0.0 c_u_constraints(carrier_production_max)(4)_: +1 variables(carrier_prod)(a__test_transmission_heat_b__heat__2005_01_01_00_00) --1 variables(energy_cap)(a__test_transmission_heat_b) -<= 0 +-1.0 variables(energy_cap)(a__test_transmission_heat_b) +<= 0.0 c_u_constraints(carrier_production_max)(5)_: +-1.0 variables(energy_cap)(a__test_transmission_heat_b) +1 variables(carrier_prod)(a__test_transmission_heat_b__heat__2005_01_01_01_00) --1 variables(energy_cap)(a__test_transmission_heat_b) -<= 0 +<= 0.0 c_u_constraints(carrier_production_max)(6)_: +1 variables(carrier_prod)(b__test_supply_elec__electricity__2005_01_01_00_00) --1 variables(energy_cap)(b__test_supply_elec) -<= 0 +-1.0 variables(energy_cap)(b__test_supply_elec) +<= 0.0 c_u_constraints(carrier_production_max)(7)_: +-1.0 variables(energy_cap)(b__test_supply_elec) +1 variables(carrier_prod)(b__test_supply_elec__electricity__2005_01_01_01_00) --1 variables(energy_cap)(b__test_supply_elec) -<= 0 +<= 0.0 c_u_constraints(carrier_production_max)(8)_: +1 variables(carrier_prod)(b__test_transmission_elec_a__electricity__2005_01_01_00_00) --1 variables(energy_cap)(b__test_transmission_elec_a) -<= 0 +-1.0 variables(energy_cap)(b__test_transmission_elec_a) +<= 0.0 c_u_constraints(carrier_production_max)(9)_: +-1.0 variables(energy_cap)(b__test_transmission_elec_a) +1 variables(carrier_prod)(b__test_transmission_elec_a__electricity__2005_01_01_01_00) --1 variables(energy_cap)(b__test_transmission_elec_a) -<= 0 +<= 0.0 c_u_constraints(carrier_production_max)(10)_: +1 variables(carrier_prod)(b__test_transmission_heat_a__heat__2005_01_01_00_00) --1 variables(energy_cap)(b__test_transmission_heat_a) -<= 0 +-1.0 variables(energy_cap)(b__test_transmission_heat_a) +<= 0.0 c_u_constraints(carrier_production_max)(11)_: +-1.0 variables(energy_cap)(b__test_transmission_heat_a) +1 variables(carrier_prod)(b__test_transmission_heat_a__heat__2005_01_01_01_00) --1 variables(energy_cap)(b__test_transmission_heat_a) -<= 0 - -c_e_ONE_VAR_CONSTANT: -ONE_VAR_CONSTANT = 1.0 +<= 0.0 bounds - 100 <= variables(energy_cap)(a__test_supply_elec) <= 100 - 0 <= variables(energy_cap)(a__test_transmission_elec_b) <= 10 - 0 <= variables(energy_cap)(a__test_transmission_heat_b) <= 5 - 0 <= variables(energy_cap)(b__test_supply_elec) <= 10 - 0 <= variables(energy_cap)(b__test_transmission_elec_a) <= 10 - 0 <= variables(energy_cap)(b__test_transmission_heat_a) <= 5 + 1 <= ONE_VAR_CONSTANT <= 1 0 <= variables(carrier_prod)(a__test_supply_elec__electricity__2005_01_01_00_00) <= +inf + 100.0 <= variables(energy_cap)(a__test_supply_elec) <= 100.0 0 <= variables(carrier_prod)(a__test_supply_elec__electricity__2005_01_01_01_00) <= +inf + 0 <= variables(carrier_prod)(a__test_transmission_elec_b__electricity__2005_01_01_00_00) <= +inf + 0.0 <= variables(energy_cap)(a__test_transmission_elec_b) <= 10.0 + 0 <= variables(carrier_prod)(a__test_transmission_elec_b__electricity__2005_01_01_01_00) <= +inf + 0 <= variables(carrier_prod)(a__test_transmission_heat_b__heat__2005_01_01_00_00) <= +inf + 0.0 <= variables(energy_cap)(a__test_transmission_heat_b) <= 5.0 + 0 <= variables(carrier_prod)(a__test_transmission_heat_b__heat__2005_01_01_01_00) <= +inf 0 <= variables(carrier_prod)(b__test_supply_elec__electricity__2005_01_01_00_00) <= +inf + 0.0 <= variables(energy_cap)(b__test_supply_elec) <= 10.0 0 <= variables(carrier_prod)(b__test_supply_elec__electricity__2005_01_01_01_00) <= +inf 0 <= variables(carrier_prod)(b__test_transmission_elec_a__electricity__2005_01_01_00_00) <= +inf + 0.0 <= variables(energy_cap)(b__test_transmission_elec_a) <= 10.0 0 <= variables(carrier_prod)(b__test_transmission_elec_a__electricity__2005_01_01_01_00) <= +inf - 0 <= variables(carrier_prod)(a__test_transmission_elec_b__electricity__2005_01_01_00_00) <= +inf - 0 <= variables(carrier_prod)(a__test_transmission_elec_b__electricity__2005_01_01_01_00) <= +inf 0 <= variables(carrier_prod)(b__test_transmission_heat_a__heat__2005_01_01_00_00) <= +inf + 0.0 <= variables(energy_cap)(b__test_transmission_heat_a) <= 5.0 0 <= variables(carrier_prod)(b__test_transmission_heat_a__heat__2005_01_01_01_00) <= +inf - 0 <= variables(carrier_prod)(a__test_transmission_heat_b__heat__2005_01_01_00_00) <= +inf - 0 <= variables(carrier_prod)(a__test_transmission_heat_b__heat__2005_01_01_01_00) <= +inf end diff --git a/calliope/test/common/lp_files/energy_cap.lp b/calliope/test/common/lp_files/energy_cap.lp index 27d072c8d..4654af5ad 100644 --- a/calliope/test/common/lp_files/energy_cap.lp +++ b/calliope/test/common/lp_files/energy_cap.lp @@ -8,9 +8,11 @@ objectives(0): s.t. c_e_ONE_VAR_CONSTANT: -ONE_VAR_CONSTANT = 1.0 ++1 ONE_VAR_CONSTANT += 1 bounds - 1 <= variables(energy_cap)(a__test_supply_elec) <= +inf - 0 <= variables(energy_cap)(b__test_supply_elec) <= 100 + 1 <= ONE_VAR_CONSTANT <= 1 + 1.0 <= variables(energy_cap)(a__test_supply_elec) <= +inf + 0.0 <= variables(energy_cap)(b__test_supply_elec) <= 100.0 end diff --git a/calliope/test/common/lp_files/resource_max.lp b/calliope/test/common/lp_files/resource_max.lp index 9c81d8058..0bda39057 100644 --- a/calliope/test/common/lp_files/resource_max.lp +++ b/calliope/test/common/lp_files/resource_max.lp @@ -2,38 +2,36 @@ min objectives(0): -+2 ONE_VAR_CONSTANT ++2.0 ONE_VAR_CONSTANT s.t. c_u_constraints(my_constraint)(0)_: --24 variables(resource_cap)(a__test_supply_plus) +1 variables(resource_con)(a__test_supply_plus__2005_01_01_00_00) -<= 0 +-24.0 variables(resource_cap)(a__test_supply_plus) +<= 0.0 c_u_constraints(my_constraint)(1)_: --24 variables(resource_cap)(a__test_supply_plus) +-24.0 variables(resource_cap)(a__test_supply_plus) +1 variables(resource_con)(a__test_supply_plus__2005_01_02_00_00) -<= 0 +<= 0.0 c_u_constraints(my_constraint)(2)_: --24 variables(resource_cap)(b__test_supply_plus) +1 variables(resource_con)(b__test_supply_plus__2005_01_01_00_00) -<= 0 +-24.0 variables(resource_cap)(b__test_supply_plus) +<= 0.0 c_u_constraints(my_constraint)(3)_: --24 variables(resource_cap)(b__test_supply_plus) +-24.0 variables(resource_cap)(b__test_supply_plus) +1 variables(resource_con)(b__test_supply_plus__2005_01_02_00_00) -<= 0 - -c_e_ONE_VAR_CONSTANT: -ONE_VAR_CONSTANT = 1.0 +<= 0.0 bounds + 1 <= ONE_VAR_CONSTANT <= 1 0 <= variables(resource_con)(a__test_supply_plus__2005_01_01_00_00) <= +inf + 0 <= variables(resource_cap)(a__test_supply_plus) <= +inf 0 <= variables(resource_con)(a__test_supply_plus__2005_01_02_00_00) <= +inf 0 <= variables(resource_con)(b__test_supply_plus__2005_01_01_00_00) <= +inf - 0 <= variables(resource_con)(b__test_supply_plus__2005_01_02_00_00) <= +inf - 0 <= variables(resource_cap)(a__test_supply_plus) <= +inf 0 <= variables(resource_cap)(b__test_supply_plus) <= +inf + 0 <= variables(resource_con)(b__test_supply_plus__2005_01_02_00_00) <= +inf end diff --git a/calliope/test/common/lp_files/storage_max.lp b/calliope/test/common/lp_files/storage_max.lp index 2dde2fe51..b23f6c6cc 100644 --- a/calliope/test/common/lp_files/storage_max.lp +++ b/calliope/test/common/lp_files/storage_max.lp @@ -9,31 +9,29 @@ s.t. c_u_constraints(storage_max)(0)_: +1 variables(storage)(a__test_storage__2005_01_01_00_00) -1 variables(storage_cap)(a__test_storage) -<= 0 +<= 0.0 c_u_constraints(storage_max)(1)_: -+1 variables(storage)(a__test_storage__2005_01_01_01_00) -1 variables(storage_cap)(a__test_storage) -<= 0 ++1 variables(storage)(a__test_storage__2005_01_01_01_00) +<= 0.0 c_u_constraints(storage_max)(2)_: +1 variables(storage)(b__test_storage__2005_01_01_00_00) -1 variables(storage_cap)(b__test_storage) -<= 0 +<= 0.0 c_u_constraints(storage_max)(3)_: -+1 variables(storage)(b__test_storage__2005_01_01_01_00) -1 variables(storage_cap)(b__test_storage) -<= 0 - -c_e_ONE_VAR_CONSTANT: -ONE_VAR_CONSTANT = 1.0 ++1 variables(storage)(b__test_storage__2005_01_01_01_00) +<= 0.0 bounds - 0 <= variables(storage_cap)(a__test_storage) <= 15 - 0 <= variables(storage_cap)(b__test_storage) <= 15 + 1 <= ONE_VAR_CONSTANT <= 1 0 <= variables(storage)(a__test_storage__2005_01_01_00_00) <= +inf + 0 <= variables(storage_cap)(a__test_storage) <= 15.0 0 <= variables(storage)(a__test_storage__2005_01_01_01_00) <= +inf 0 <= variables(storage)(b__test_storage__2005_01_01_00_00) <= +inf + 0 <= variables(storage_cap)(b__test_storage) <= 15.0 0 <= variables(storage)(b__test_storage__2005_01_01_01_00) <= +inf -end +end \ No newline at end of file diff --git a/conftest.py b/calliope/test/conftest.py similarity index 100% rename from conftest.py rename to calliope/test/conftest.py diff --git a/calliope/test/test_backend_expression_parser.py b/calliope/test/test_backend_expression_parser.py index 9b32289a0..7e939a984 100644 --- a/calliope/test/test_backend_expression_parser.py +++ b/calliope/test/test_backend_expression_parser.py @@ -727,7 +727,9 @@ def test_addition_multiplication( if np.isinf(float1) and np.isinf(float2) and sign in ["-", "/"]: assert np.isnan(evaluated_) else: - assert evaluated_ == getattr(operator, sign_name)(float1, float2) + assert evaluated_ == pytest.approx( + getattr(operator, sign_name)(float1, float2) + ) @pytest.mark.parametrize(["sign", "sign_name"], [("+", "pos"), ("-", "neg")]) @pytest.mark.parametrize( diff --git a/calliope/test/test_backend_latex_backend.py b/calliope/test/test_backend_latex_backend.py index f64afb77c..183b45e51 100644 --- a/calliope/test/test_backend_latex_backend.py +++ b/calliope/test/test_backend_latex_backend.py @@ -421,9 +421,9 @@ def test_render(self, dummy_latex_backend_model, instring, kwargs, expected): def test_get_capacity_bounds(self, dummy_latex_backend_model, dummy_model_data): bounds = {"min": 1, "max": 2e6} lb, ub = dummy_latex_backend_model._get_capacity_bounds( - bounds, "var", dummy_model_data + bounds, "multi_dim_var", dummy_model_data ) - assert lb == {"expression": r"1 \leq \textbf{var}_\text{node,tech}"} + assert lb == {"expression": r"1 \leq \textbf{multi_dim_var}_\text{node,tech}"} assert ub == { - "expression": r"\textbf{var}_\text{node,tech} \leq 2\mathord{\times}10^{+06}" + "expression": r"\textbf{multi_dim_var}_\text{node,tech} \leq 2\mathord{\times}10^{+06}" } diff --git a/calliope/test/test_core_preprocess.py b/calliope/test/test_core_preprocess.py index baccbb7b2..58e3c5435 100644 --- a/calliope/test/test_core_preprocess.py +++ b/calliope/test/test_core_preprocess.py @@ -242,9 +242,8 @@ def test_incorrect_subset_time(self): string/integer """ - override = lambda param: AttrDict.from_yaml_string( - "model.subset_time: {}".format(param) - ) + def override(param): + return AttrDict.from_yaml_string("model.subset_time: {}".format(param)) # should fail: one string in list with pytest.raises(exceptions.ModelError): @@ -1003,13 +1002,18 @@ def test_exporting_unspecified_carrier(self): User can only define an export carrier if it is defined in ['carrier_out', 'carrier_out_2', 'carrier_out_3'] """ - override_supply = lambda param: AttrDict.from_yaml_string( - "techs.test_supply_elec.constraints.export_carrier: {}".format(param) - ) - override_converison_plus = lambda param: AttrDict.from_yaml_string( - "techs.test_conversion_plus.constraints.export_carrier: {}".format(param) - ) + def override_supply(param): + return AttrDict.from_yaml_string( + "techs.test_supply_elec.constraints.export_carrier: {}".format(param) + ) + + def override_converison_plus(param): + return AttrDict.from_yaml_string( + "techs.test_conversion_plus.constraints.export_carrier: {}".format( + param + ) + ) # should fail: exporting `heat` not allowed for electricity supply tech with pytest.raises(exceptions.ModelError): @@ -1116,9 +1120,12 @@ def test_allowed_time_varying_constraints(self): ) ) - override = lambda param: AttrDict.from_yaml_string( - "techs.test_storage.constraints.{}: file=binary_one_day.csv".format(param) - ) + def override(param): + return AttrDict.from_yaml_string( + "techs.test_storage.constraints.{}: file=binary_one_day.csv".format( + param + ) + ) # should fail: Cannot have `file=` on the following constraints for param in allowed_constraints_no_file: diff --git a/calliope/test/test_core_time.py b/calliope/test/test_core_time.py index 42062d133..36f47a4d3 100644 --- a/calliope/test/test_core_time.py +++ b/calliope/test/test_core_time.py @@ -72,6 +72,7 @@ def test_hierarchical_closest(self, model_national): # FIXME + @pytest.mark.time_intensive def test_hartigans_rule(self, model_national): data = model_national._model_data diff --git a/calliope/test/test_example_models.py b/calliope/test/test_example_models.py index 28825faf7..7d73bf052 100755 --- a/calliope/test/test_example_models.py +++ b/calliope/test/test_example_models.py @@ -14,6 +14,7 @@ class TestModelPreproccessing: def test_preprocess_national_scale(self): calliope.examples.national_scale() + @pytest.mark.time_intensive def test_preprocess_time_clustering(self): calliope.examples.time_clustering() @@ -412,6 +413,7 @@ def test_nationalscale_resampled_example_results_glpk(self): pytest.skip("GLPK not installed") +@pytest.mark.time_intensive class TestNationalScaleClusteredExampleModelSenseChecks: def model_runner( self, @@ -436,7 +438,7 @@ def model_runner( } if storage is False: override.update({"techs.battery.exists": False, "techs.csp.exists": False}) - if solver_io: + if solver_io is not None: override["run.solver_io"] = solver_io if storage_inter_cluster and backend_runner == "solve": override["model.custom_math"] = ["storage_inter_cluster"] diff --git a/calliope/test/test_math.py b/calliope/test/test_math.py index cbd1dc328..bb45860d2 100644 --- a/calliope/test/test_math.py +++ b/calliope/test/test_math.py @@ -1,8 +1,8 @@ -import filecmp from pathlib import Path import numpy as np import pytest +from pyomo.repn.tests import lp_diff import calliope from calliope import AttrDict @@ -18,8 +18,10 @@ def _compare_lps(model, custom_math, filename): expected_file = ( Path(calliope.__file__).parent / "test" / "common" / "lp_files" / lp_file ) - - assert filecmp.cmp(generated_file, expected_file) + diff = lp_diff.load_and_compare_lp_baseline( + generated_file.as_posix(), expected_file.as_posix() + ) + assert diff == ([], []) return _compare_lps diff --git a/calliope/time/clustering.py b/calliope/time/clustering.py index b0d0f6a04..c26bfd416 100644 --- a/calliope/time/clustering.py +++ b/calliope/time/clustering.py @@ -22,7 +22,10 @@ def _stack_data(data, dates, times): Stack all non-time dimensions of an xarray DataArray """ data_to_stack = data.assign_coords( - timesteps=pd.MultiIndex.from_product([dates, times], names=["dates", "times"]) + xr.Coordinates.from_pandas_multiindex( + pd.MultiIndex.from_product([dates, times], names=["dates", "times"]), + "timesteps", + ) ).unstack("timesteps") non_date_dims = list(set(data_to_stack.dims).difference(["dates", "times"])) + [ "times" @@ -270,7 +273,7 @@ def map_clusters_to_data( timesteps_per_day = len(daily_timesteps) idx = clusters.index new_idx = _timesteps_from_daily_index(idx, daily_timesteps) - clusters_timeseries = clusters.reindex(new_idx).fillna(method="ffill").astype(int) + clusters_timeseries = clusters.reindex(new_idx).ffill().astype(int) new_data = get_mean_from_clusters(data, clusters_timeseries, timesteps_per_day) new_data.attrs = data.attrs @@ -312,9 +315,10 @@ def map_clusters_to_data( "closest day".format(cluster_diff) ) timestamps = timestamps.drop_duplicates() + clusters = pd.Series(index=clusterdays_timeseries.index, dtype=int) for cluster, date in timestamps.items(): - clusterdays_timeseries.loc[clusterdays_timeseries == date] = cluster - clusters = clusterdays_timeseries.astype(int).resample("1D").mean() + clusters.loc[clusterdays_timeseries == date] = cluster + clusters = clusters.resample("1D").mean() _clusters = xr.DataArray( data=np.full(len(new_data.timesteps.values), np.nan), @@ -328,7 +332,7 @@ def map_clusters_to_data( new_data["timestep_cluster"] = _clusters.astype(int) weights = value_counts.reindex( _timesteps_from_daily_index(value_counts.index, daily_timesteps) - ).fillna(method="ffill") + ).ffill() new_data["timestep_weights"] = xr.DataArray(weights, dims=["timesteps"]) days = np.unique(new_data.timesteps.to_index().date) new_data["timestep_resolution"] = xr.DataArray( diff --git a/calliope/time/funcs.py b/calliope/time/funcs.py index 0970af13b..51b2dbd07 100644 --- a/calliope/time/funcs.py +++ b/calliope/time/funcs.py @@ -159,11 +159,12 @@ def apply_clustering( for dim in data_to_cluster.dims: data_to_cluster[dim] = data[dim] - with pd.option_context("mode.use_inf_as_na", True): - if normalize: - data_normalized = normalized_copy(data_to_cluster) - else: - data_normalized = data_to_cluster + + data_to_cluster = data_to_cluster.where(~np.isinf(data_to_cluster)) + if normalize: + data_normalized = normalized_copy(data_to_cluster) + else: + data_normalized = data_to_cluster if "file=" in clustering_func: file = clustering_func.split("=")[1] diff --git a/doc/_static/notebooks/calliope_model_object.ipynb b/doc/_static/notebooks/calliope_model_object.ipynb index 2f41b9c89..2a3f7bbd7 100644 --- a/doc/_static/notebooks/calliope_model_object.ipynb +++ b/doc/_static/notebooks/calliope_model_object.ipynb @@ -6,13 +6,11 @@ "metadata": {}, "outputs": [], "source": [ - "import xarray as xr\n", "import pandas as pd\n", - "import numpy as np\n", "\n", "import calliope\n", "\n", - "calliope.set_log_verbosity('INFO', include_solver_output=False)" + "calliope.set_log_verbosity(\"INFO\", include_solver_output=False)" ] }, { @@ -127,7 +125,7 @@ "source": [ "# All locations now hold all information about a technology at that location\n", "\n", - "m._model_run['locations']['X2']['techs']['pv']" + "m._model_run[\"locations\"][\"X2\"][\"techs\"][\"pv\"]" ] }, { @@ -162,7 +160,7 @@ "source": [ "# This includes location-specific overrides, such as energy_cap_max of 50 for the pv technology at location X3\n", "\n", - "m._model_run['locations']['X3']['techs']['pv']" + "m._model_run[\"locations\"][\"X3\"][\"techs\"][\"pv\"]" ] }, { @@ -208,10 +206,10 @@ ], "source": [ "# All sets have also been collated.\n", - "# locations and technologies are concatenated into loc::tech sets, \n", + "# locations and technologies are concatenated into loc::tech sets,\n", "# to create a dense matrix and smaller overall model size\n", "\n", - "m._model_run['sets']['loc_techs']" + "m._model_run[\"sets\"][\"loc_techs\"]" ] }, { @@ -256,10 +254,10 @@ } ], "source": [ - "# For every constraint, a set of loc_techs (or loc_tech_carriers) is prepared, \n", + "# For every constraint, a set of loc_techs (or loc_tech_carriers) is prepared,\n", "# so we only build the constraint over that set\n", "\n", - "m._model_run['constraint_sets']['loc_techs_energy_capacity_constraint']" + "m._model_run[\"constraint_sets\"][\"loc_techs_energy_capacity_constraint\"]" ] }, { @@ -279,7 +277,7 @@ } ], "source": [ - "m._model_run['constraint_sets']['loc_techs_resource_area_constraint']" + "m._model_run[\"constraint_sets\"][\"loc_techs_resource_area_constraint\"]" ] }, { @@ -358,7 +356,7 @@ ], "source": [ "# timeseries data is stored as dataframes, having been loaded from CSV\n", - "m._model_run['timeseries_data']['pv_resource.csv'].head()" + "m._model_run[\"timeseries_data\"][\"pv_resource.csv\"].head()" ] }, { @@ -1144,7 +1142,7 @@ } ], "source": [ - "# Users would usually access information for the initialised model using m.inputs \n", + "# Users would usually access information for the initialised model using m.inputs\n", "m.inputs" ] }, @@ -2929,7 +2927,7 @@ } ], "source": [ - "# If timeseries aggregation of any kind has taken place, then m._model_data_original can be accessed to see the \n", + "# If timeseries aggregation of any kind has taken place, then m._model_data_original can be accessed to see the\n", "# model data prior to aggregation\n", "m._model_data_original # In this case, it is the same as m._model_data" ] @@ -3316,7 +3314,7 @@ ], "source": [ "# We can find the same PV energy_cap_max data as seen in m._model_run\n", - "m._model_data.energy_cap_max.loc[{'loc_techs': 'X2::pv'}]" + "m._model_data.energy_cap_max.loc[{\"loc_techs\": \"X2::pv\"}]" ] }, { @@ -3700,7 +3698,7 @@ } ], "source": [ - "m._model_data.energy_cap_max.loc[{'loc_techs': 'X3::pv'}]" + "m._model_data.energy_cap_max.loc[{\"loc_techs\": \"X3::pv\"}]" ] }, { @@ -5833,7 +5831,7 @@ "source": [ "# Data can also be reformatted to be easier to read (removes dimension concatenation).\n", "# Conversion to a pandas DataFrame is a good idea for greater readibility.\n", - "m.get_formatted_array('energy_cap').to_pandas()" + "m.get_formatted_array(\"energy_cap\").to_pandas()" ] }, { @@ -5867,7 +5865,9 @@ "source": [ "# >2 dimensions cannot be easily viewed in a pandas dataframe, unless a MultiIndex is used.\n", "# To view a 4-dimensional result, we can use `to_series()`\n", - "m.get_formatted_array('carrier_prod').to_series().dropna() # drop_na() removes all NaN values" + "m.get_formatted_array(\n", + " \"carrier_prod\"\n", + ").to_series().dropna() # drop_na() removes all NaN values" ] }, { @@ -6245,9 +6245,14 @@ "source": [ "# The inputs as used by Pyomo can be printed. This includes filled default data where necessary\n", "pd.concat(\n", - " (m.backend.access_model_inputs()['energy_cap_max'].to_pandas().rename('backend'), # get the data from Pyomo\n", - " m.inputs['energy_cap_max'].to_pandas().rename('pre-run')), # get the data from model_data (via inputs)\n", - " axis=1, sort=True\n", + " (\n", + " m.backend.access_model_inputs()[\"energy_cap_max\"]\n", + " .to_pandas()\n", + " .rename(\"backend\"), # get the data from Pyomo\n", + " m.inputs[\"energy_cap_max\"].to_pandas().rename(\"pre-run\"),\n", + " ), # get the data from model_data (via inputs)\n", + " axis=1,\n", + " sort=True,\n", ")" ] }, @@ -6288,7 +6293,9 @@ "source": [ "# We can activate and deactivate constraints, such as switching off the energy capacity constraint\n", "\n", - "m.backend.activate_constraint('energy_capacity_constraint', False) # set to True to activate\n", + "m.backend.activate_constraint(\n", + " \"energy_capacity_constraint\", False\n", + ") # set to True to activate\n", "m._backend_model.energy_capacity_constraint.pprint()" ] }, @@ -6321,7 +6328,7 @@ } ], "source": [ - "# Rerun the model with this constraint switched off. \n", + "# Rerun the model with this constraint switched off.\n", "# This will dump results to a new dataset, *NOT* to m._model_data (or m.results)\n", "new_model = m.backend.rerun()" ] @@ -6534,8 +6541,14 @@ ], "source": [ "# The results are now updated, which we can compare to our old results\n", - "pd.concat((new_model.results.energy_cap.to_pandas().rename('new'), m.results.energy_cap.to_pandas().rename('old')), \n", - " axis=1, sort=True)" + "pd.concat(\n", + " (\n", + " new_model.results.energy_cap.to_pandas().rename(\"new\"),\n", + " m.results.energy_cap.to_pandas().rename(\"old\"),\n", + " ),\n", + " axis=1,\n", + " sort=True,\n", + ")" ] }, { @@ -6579,7 +6592,7 @@ } ], "source": [ - "# We can also see that the Pyomo backend_model has updated to the new values \n", + "# We can also see that the Pyomo backend_model has updated to the new values\n", "m._backend_model.energy_cap.pprint()" ] }, @@ -7611,8 +7624,8 @@ "# We can save at any point, which will dump the entire m._model_data to file.\n", "# NetCDF is recommended, as it retains most of the data and can be reloaded into a Calliope model at a later date.\n", "\n", - "m.to_netcdf('path/to/file.nc') # Saves a single file\n", - "m.to_csv('path/to/folder') # Saves a file for each xarray DataArray" + "m.to_netcdf(\"path/to/file.nc\") # Saves a single file\n", + "m.to_csv(\"path/to/folder\") # Saves a file for each xarray DataArray" ] } ], diff --git a/doc/_static/notebooks/milp.ipynb b/doc/_static/notebooks/milp.ipynb index 5e524be4f..ce1dd0c07 100644 --- a/doc/_static/notebooks/milp.ipynb +++ b/doc/_static/notebooks/milp.ipynb @@ -18,7 +18,7 @@ "import calliope\n", "\n", "# We increase logging verbosity\n", - "calliope.set_log_verbosity('INFO', include_solver_output=False)" + "calliope.set_log_verbosity(\"INFO\", include_solver_output=False)" ] }, { @@ -845,10 +845,10 @@ } ], "source": [ - "# Model inputs can be viewed at `model.inputs`. \n", - "# Variables are indexed over any combination of `techs`, `locs`, `carriers`, `costs` and `timesteps`, \n", - "# although `techs`, `locs`, and `carriers` are often concatenated. \n", - "# e.g. `chp`, `X1`, `heat` -> `X1::chp::heat` \n", + "# Model inputs can be viewed at `model.inputs`.\n", + "# Variables are indexed over any combination of `techs`, `locs`, `carriers`, `costs` and `timesteps`,\n", + "# although `techs`, `locs`, and `carriers` are often concatenated.\n", + "# e.g. `chp`, `X1`, `heat` -> `X1::chp::heat`\n", "model.inputs" ] }, @@ -955,7 +955,7 @@ } ], "source": [ - "# Solve the model. Results are loaded into `model.results`. \n", + "# Solve the model. Results are loaded into `model.results`.\n", "# By including logging (see package importing), we can see the timing of parts of the run, as well as the solver's log\n", "model.run()" ] @@ -2054,7 +2054,7 @@ } ], "source": [ - "# Model results are held in the same structure as model inputs. \n", + "# Model results are held in the same structure as model inputs.\n", "# The results consist of the optimal values for all decision variables, including capacities and carrier flow\n", "# There are also results, like system capacity factor and levelised costs, which are calculated in postprocessing\n", "# before being added to the results Dataset\n", @@ -2084,9 +2084,9 @@ ], "source": [ "# We can sum operating units of CHP over all locations and turn the result into a pandas DataFrame\n", - "df_units = model.get_formatted_array('operating_units').sum('locs').to_pandas().T\n", + "df_units = model.get_formatted_array(\"operating_units\").sum(\"locs\").to_pandas().T\n", "\n", - "#The information about the dataframe tells us about the amount of data it holds in the index and in each column\n", + "# The information about the dataframe tells us about the amount of data it holds in the index and in each column\n", "df_units.info()" ] }, @@ -6626,7 +6626,7 @@ "source": [ "# plot.capacities gives a graphical view of the non-timeseries variables, both input and output\n", "\n", - "# Note, because we fix unit size, CHP now has a maximum capacity of 300kW, \n", + "# Note, because we fix unit size, CHP now has a maximum capacity of 300kW,\n", "# compared to 260kW in the non-MILP case\n", "\n", "model.plot.capacity()" diff --git a/doc/_static/notebooks/national_scale.ipynb b/doc/_static/notebooks/national_scale.ipynb index 434e69ae5..6593b71a7 100644 --- a/doc/_static/notebooks/national_scale.ipynb +++ b/doc/_static/notebooks/national_scale.ipynb @@ -16,7 +16,7 @@ "import calliope\n", "\n", "# We increase logging verbosity\n", - "calliope.set_log_verbosity('INFO', include_solver_output=False)" + "calliope.set_log_verbosity(\"INFO\", include_solver_output=False)" ] }, { @@ -801,10 +801,10 @@ } ], "source": [ - "# Model inputs can be viewed at `model.inputs`. \n", - "# Variables are indexed over any combination of `techs`, `locs`, `carriers`, `costs` and `timesteps`, \n", - "# although `techs`, `locs`, and `carriers` are often concatenated. \n", - "# e.g. `ccgt`, `region1`, `power` -> `region1::ccgt::power` \n", + "# Model inputs can be viewed at `model.inputs`.\n", + "# Variables are indexed over any combination of `techs`, `locs`, `carriers`, `costs` and `timesteps`,\n", + "# although `techs`, `locs`, and `carriers` are often concatenated.\n", + "# e.g. `ccgt`, `region1`, `power` -> `region1::ccgt::power`\n", "model.inputs" ] }, @@ -1274,8 +1274,8 @@ ], "source": [ "# To reformat the array, deconcatenating loc_techs / loc_tech_carriers, you can use model.get_formatted_array()\n", - "# You can then apply loc/tech/carrier only operations, like summing information over locations: \n", - "model.get_formatted_array('resource').sum('locs').to_pandas()" + "# You can then apply loc/tech/carrier only operations, like summing information over locations:\n", + "model.get_formatted_array(\"resource\").sum(\"locs\").to_pandas()" ] }, { @@ -1316,7 +1316,7 @@ } ], "source": [ - "# Solve the model. Results are loaded into `model.results`. \n", + "# Solve the model. Results are loaded into `model.results`.\n", "# By including logging (see package importing), we can see the timing of parts of the run, as well as the solver's log\n", "model.run()" ] @@ -2328,7 +2328,7 @@ } ], "source": [ - "# Model results are held in the same structure as model inputs. \n", + "# Model results are held in the same structure as model inputs.\n", "# The results consist of the optimal values for all decision variables, including capacities and carrier flow\n", "# There are also results, like system capacity factor and levelised costs, which are calculated in postprocessing\n", "# before being added to the results Dataset\n", @@ -2366,9 +2366,15 @@ ], "source": [ "# We can sum power output over all locations and turn the result into a pandas DataFrame\n", - "df_power = model.get_formatted_array('carrier_prod').loc[{'carriers':'power'}].sum('locs').to_pandas().T\n", + "df_power = (\n", + " model.get_formatted_array(\"carrier_prod\")\n", + " .loc[{\"carriers\": \"power\"}]\n", + " .sum(\"locs\")\n", + " .to_pandas()\n", + " .T\n", + ")\n", "\n", - "#The information about the dataframe tells us about the amount of data it holds in the index and in each column\n", + "# The information about the dataframe tells us about the amount of data it holds in the index and in each column\n", "df_power.info()" ] }, @@ -8302,10 +8308,10 @@ ], "source": [ "# We can also examine total technology costs\n", - "# notice all the NaN values which appear when seperating loc::techs to locs and techs. \n", + "# notice all the NaN values which appear when seperating loc::techs to locs and techs.\n", "# Any NaN value means we never considered that combination of `loc` and `tech` for the variable\n", "\n", - "costs = model.get_formatted_array('cost').loc[{'costs': 'monetary'}].to_pandas()\n", + "costs = model.get_formatted_array(\"cost\").loc[{\"costs\": \"monetary\"}].to_pandas()\n", "costs" ] }, @@ -8341,7 +8347,9 @@ "source": [ "# We can examine levelized costs for each location and technology\n", "\n", - "lcoes = model.results.systemwide_levelised_cost.loc[{'carriers': 'power', 'costs':'monetary'}].to_pandas()\n", + "lcoes = model.results.systemwide_levelised_cost.loc[\n", + " {\"carriers\": \"power\", \"costs\": \"monetary\"}\n", + "].to_pandas()\n", "lcoes" ] }, @@ -8535,7 +8543,7 @@ ], "source": [ "# We can just plot this directly using calliope analysis functionality\n", - "model.plot.capacity(array='systemwide_levelised_cost')" + "model.plot.capacity(array=\"systemwide_levelised_cost\")" ] }, { @@ -24832,7 +24840,7 @@ } ], "source": [ - "# Transmission plots give us a static picture of installed capacity along links. \n", + "# Transmission plots give us a static picture of installed capacity along links.\n", "# If we want timeseries data on energy flows at locations and along links\n", "# we can use energy flow plotting. We only show every 4th hour here, to improve loading speed.\n", "\n", diff --git a/doc/_static/notebooks/urban_scale.ipynb b/doc/_static/notebooks/urban_scale.ipynb index 51183d20f..37a1708c4 100644 --- a/doc/_static/notebooks/urban_scale.ipynb +++ b/doc/_static/notebooks/urban_scale.ipynb @@ -16,7 +16,7 @@ "import calliope\n", "\n", "# We increase logging verbosity\n", - "calliope.set_log_verbosity('INFO', include_solver_output=False)" + "calliope.set_log_verbosity(\"INFO\", include_solver_output=False)" ] }, { @@ -819,10 +819,10 @@ } ], "source": [ - "# Model inputs can be viewed at `model.inputs`. \n", - "# Variables are indexed over any combination of `techs`, `locs`, `carriers`, `costs` and `timesteps`, \n", - "# although `techs`, `locs`, and `carriers` are often concatenated. \n", - "# e.g. `chp`, `X1`, `heat` -> `X1::chp::heat` \n", + "# Model inputs can be viewed at `model.inputs`.\n", + "# Variables are indexed over any combination of `techs`, `locs`, `carriers`, `costs` and `timesteps`,\n", + "# although `techs`, `locs`, and `carriers` are often concatenated.\n", + "# e.g. `chp`, `X1`, `heat` -> `X1::chp::heat`\n", "model.inputs" ] }, @@ -1477,8 +1477,8 @@ ], "source": [ "# To reformat the array, deconcatenating loc_techs / loc_tech_carriers, you can use model.get_formatted_array()\n", - "# You can then apply loc/tech/carrier only operations, like summing information over locations: \n", - "model.get_formatted_array('resource').sum('locs').to_pandas()" + "# You can then apply loc/tech/carrier only operations, like summing information over locations:\n", + "model.get_formatted_array(\"resource\").sum(\"locs\").to_pandas()" ] }, { @@ -1519,7 +1519,7 @@ } ], "source": [ - "# Solve the model. Results are loaded into `model.results`. \n", + "# Solve the model. Results are loaded into `model.results`.\n", "# By including logging (see package importing), we can see the timing of parts of the run, as well as the solver's log\n", "model.run()" ] @@ -2590,7 +2590,7 @@ } ], "source": [ - "# Model results are held in the same structure as model inputs. \n", + "# Model results are held in the same structure as model inputs.\n", "# The results consist of the optimal values for all decision variables, including capacities and carrier flow\n", "# There are also results, like system capacity factor and levelised costs, which are calculated in postprocessing\n", "# before being added to the results Dataset\n", @@ -2631,9 +2631,15 @@ ], "source": [ "# We can sum heat output over all locations and turn the result into a pandas DataFrame\n", - "df_heat = model.get_formatted_array('carrier_prod').loc[{'carriers':'heat'}].sum('locs').to_pandas().T\n", + "df_heat = (\n", + " model.get_formatted_array(\"carrier_prod\")\n", + " .loc[{\"carriers\": \"heat\"}]\n", + " .sum(\"locs\")\n", + " .to_pandas()\n", + " .T\n", + ")\n", "\n", - "#The information about the dataframe tells us about the amount of data it holds in the index and in each column\n", + "# The information about the dataframe tells us about the amount of data it holds in the index and in each column\n", "df_heat.info()" ] }, @@ -7157,10 +7163,10 @@ ], "source": [ "# We can also examine total technology costs\n", - "# notice all the NaN values which appear when seperating loc::techs to locs and techs. \n", + "# notice all the NaN values which appear when seperating loc::techs to locs and techs.\n", "# Any NaN value means we never considered that combination of `loc` and `tech` for the variable\n", "\n", - "costs = model.get_formatted_array('cost').loc[{'costs': 'monetary'}].to_pandas()\n", + "costs = model.get_formatted_array(\"cost\").loc[{\"costs\": \"monetary\"}].to_pandas()\n", "costs" ] }, @@ -7200,7 +7206,9 @@ "source": [ "# We can examine levelized costs for each location and technology\n", "\n", - "lcoes = model.results.systemwide_levelised_cost.loc[{'carriers': 'electricity', 'costs':'monetary'}].to_pandas()\n", + "lcoes = model.results.systemwide_levelised_cost.loc[\n", + " {\"carriers\": \"electricity\", \"costs\": \"monetary\"}\n", + "].to_pandas()\n", "lcoes" ] }, @@ -7452,7 +7460,7 @@ ], "source": [ "# We can just plot this directly using calliope analysis functionality\n", - "model.plot.capacity(array='systemwide_levelised_cost')" + "model.plot.capacity(array=\"systemwide_levelised_cost\")" ] }, { @@ -31860,7 +31868,7 @@ } ], "source": [ - "# Transmission plots give us a static picture of installed capacity along links. \n", + "# Transmission plots give us a static picture of installed capacity along links.\n", "# If we want timeseries data on energy flows at locations and along links\n", "# we can use energy flow plotting. We only show the first day here, to improve loading speed.\n", "\n", diff --git a/doc/conf.py b/doc/conf.py index cb184d14f..999dd7273 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -103,7 +103,7 @@ # Mock modules for Read The Docs autodoc generation -MOCK_MODULES = ["xarray", "pandas", "numpy", "pyomo", "sklearn", "pyparsing"] +MOCK_MODULES = ["xarray", "pandas", "numpy", "pyomo", "sklearn", "pyparsing", "netCDF4"] autodoc_mock_imports = MOCK_MODULES autodoc_typehints = "both" autodoc_member_order = "bysource" diff --git a/doc/helpers/generate_math.py b/doc/helpers/generate_math.py index ec683e01d..0d076931b 100644 --- a/doc/helpers/generate_math.py +++ b/doc/helpers/generate_math.py @@ -66,8 +66,9 @@ def generate_custom_math_model( def generate_model_config() -> dict[str, dict]: - """ - To generate the written mathematical formulation of all possible base constraints, we first create a dummy model that has all the relevant technology groups defining all their allowed parameters defined. + """To generate the written mathematical formulation of all possible base constraints, we first create a dummy model. + + This dummy has all the relevant technology groups defining all their allowed parameters defined. Parameters that can be defined over a timeseries are forced to be defined over a timeseries. Accordingly, the parameters will have "timesteps" in their dimensions in the formulation. """ diff --git a/doc/user/develop.rst b/doc/user/develop.rst index 442fbbcc9..d1a22bce1 100644 --- a/doc/user/develop.rst +++ b/doc/user/develop.rst @@ -32,13 +32,12 @@ Then install all development requirements for Calliope into a new environment, c $ cd calliope $ conda config --add channels conda-forge # since we cannot explicitly request it with `conda env update`, we add the `conda-forge` package channel to the user's conda configuration file. - $ conda create -n calliope_dev python=3.9 # to ensure the correct python version is installed + $ conda create -n calliope_dev python=3.11 # to ensure the correct python version is installed $ conda env update -f requirements.yml -n calliope_dev # to install the calliope non-python dependencies and testing/coverage python packages $ conda env update -f requirements.txt -n calliope_dev # to install the pinned calliope python dependencies $ conda activate calliope_dev - $ pip install -e . # installs from your local clone of the calliope repository + $ pip install --no-deps -e . # installs from your local clone of the calliope repository -Only calliope itself should be installed from pip, the rest will have been installed from conda and will be marked as `Requirement already satisfied` on running the above command. .. Note:: Most of our tests depend on having the CBC solver also installed, as we have found it to be more stable than GPLK. If you are running on a Unix system, then you can run ``conda install coincbc`` to also install the CBC solver. To install solvers other than CBC, and for Windows systems, see our :ref:`solver installation instructions `. @@ -148,13 +147,47 @@ Finally, push the branch online, so it's existence is also in your remote fork o $ git push -u origin new-fix-or-feature -Now the files in your local directory can be edited with complete freedom. Once you have made the necessary changes, you'll need to test that they don't break anything. This can be done easily by changing to the directory into which you cloned your fork using the terminal / command line, and running `pytest `_ (make sure you have activated the conda environment and you have pytest installed: `conda install pytest`). Any change you make should also be covered by a test. Add it into the relevant test file, making sure the function starts with 'test\_'. Since the whole test suite takes ~25 minutes to run, you can run specific tests, such as those you add in +Now the files in your local directory can be edited with complete freedom. Once you have made the necessary changes, you'll need to test that they don't break anything. This can be done easily by changing to the directory into which you cloned your fork using the terminal / command line, and running `pytest `_ (make sure you have activated the conda environment and you have pytest installed: `conda install pytest`). Any change you make should also be covered by a test. Add it into the relevant test file, making sure the function starts with 'test\_'. - .. code-block:: fishshell +If tests are failing, you can debug them by using the pytest arguments ``-x`` (stop at the first failed test) and ``--pdb`` (enter into the debug console). - $ pytest calliope/test/test_filename.py::ClassName::function_name +Speeding up your tests +^^^^^^^^^^^^^^^^^^^^^^ +When making a change you may need to run your tests multiple times until you have made the relevant code changes for everything to pass; remember that *all* tests need to pass, not only those you've recently added. +Since the whole test suite takes ~25 minutes to run, you can do some things to speed up the process: -If tests are failing, you can debug them by using the pytest arguments ``-x`` (stop at the first failed test) and ``--pdb`` (enter into the debug console). +#. parallelise tests + If you have installed the calliope development environment, it includes the `pytest-xdist `_ package, which allows you to run tests in parallel. + To do so with calliope with the maximum available threads on your device, run: + + .. code-block:: fishshell + + $ pytest -n=auto --dist=loadscope + + `--dist=loadscope` ensures that _within_ test classes, the tests run in series, since some might depend on changes made to `fixtures `_ by previous tests. + +#. run specific tests. + E.g., for `test_function_name` inside `test_filename.py` and under the class name `TestClassName`, you would run: + + .. code-block:: fishshell + + $ pytest calliope/test/test_filename.py::TestClassName::test_function_name + + You can also run only those tests that previously failed with `--lf`: + + .. code-block:: fishshell + + $ pytest --lf + +#. Avoid time intensive tests. + You can run most of the test suite but avoid running the more time intensive tests (which are mostly concerned with timeseries clustering) by activating the following pytest marker: + + .. code-block:: fishshell + + $ pytest -m "not time_intensive" + +Contributing a change +--------------------- Once everything has been updated as you'd like (see the contribution checklist below for more on this), you can commit those changes. This stores all edited files in the directory, ready for pushing online diff --git a/doc/user/installation.rst b/doc/user/installation.rst index b9c2d3086..741038b1a 100644 --- a/doc/user/installation.rst +++ b/doc/user/installation.rst @@ -11,7 +11,7 @@ Calliope has been tested on Linux, macOS, and Windows. Running Calliope requires four things: -1. The Python programming language, version 3.8 or 3.9. +1. The Python programming language, version 3.9 to 3.11. 2. A number of Python add-on modules (see :ref:`below for the complete list `). 3. A solver: Calliope has been tested with CBC, GLPK, Gurobi, and CPLEX. Any other solver that is compatible with Pyomo should also work. 4. The Calliope software itself. @@ -29,8 +29,7 @@ With Miniconda installed, you can create a new environment called ``"calliope"`` $ conda create -c conda-forge -n calliope calliope -This will install calliope with Python version 3.9. -Due to incompatabilities between packages that Calliope relies upon, it is only possible to install this version of Calliope on Python versions 3.8 and 3.9. +This will install calliope with Python version 3.11. To use Calliope, you need to activate the ``calliope`` environment each time diff --git a/pyproject.toml b/pyproject.toml index c61d33102..fa2fecef2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,5 +22,27 @@ exclude = ''' )/ ''' -[tool.isort] -profile = "black" \ No newline at end of file +[tool.ruff] +line-length = 88 +select = ["E", "F", "I", "Q", "W"] +# line too long; Black will handle these +ignore = ["E501"] + +[tool.ruff.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 + +# Ignore `E402` (import violations) and `F401` (unused imports) in all `__init__.py` files +[tool.ruff.per-file-ignores] +"__init__.py" = ["E402", "F401"] +"*.ipynb" = ["E402"] + +[tool.ruff.flake8-quotes] +docstring-quotes = "double" + +[tool.ruff.pydocstyle] +convention = "google" + +[tool.ruff.pycodestyle] +max-doc-length = 200 +ignore-overlong-task-comments = true diff --git a/pytest.ini b/pytest.ini index 3a3f9d762..a829721e8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,4 +8,5 @@ filterwarnings = ignore:.*distutils Version classes are deprecated.*:DeprecationWarning: ignore:.*`np.float` is a deprecated alias.*:DeprecationWarning: markers = - serial \ No newline at end of file + serial + time_intensive \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3df4eadd3..a11945da0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -click ~= 8.1 -ipython >= 7, < 9 -ipdb >= 0.13, < 1.0 -jinja2 ~= 3.1.2 -jsonschema ~= 4.17.3 -natsort >= 5, < 9 -netcdf4 >= 1.2.2, < 1.7 -numpy ~= 1.23.5 -pandas ~= 1.5.2 -pyomo ~= 6.4.4 -pyparsing ~= 3.0.9 -ruamel.yaml ~= 0.17.21 -scikit-learn ~= 1.2.0 -xarray ~= 2022.3.0 +click >= 8, < 9 +ipython >= 8, < 9 +ipdb >= 0.13, < 0.14 +jinja2 >= 3, < 4 +jsonschema >= 4.17, < 4.19 +natsort >= 8, < 9 +netcdf4 >= 1.2, < 1.7 +numpy >= 1, < 2 +pandas >= 2, < 3 +pyomo >= 6.5, < 7 +pyparsing >= 3.0, < 3.1 +ruamel.yaml >= 0.17, < 0.18 +scikit-learn >= 1.2, < 1.3 +xarray >= 2023.9, < 2024.2 diff --git a/requirements_docs.yml b/requirements_docs.yml index ba06002a1..08a298e45 100644 --- a/requirements_docs.yml +++ b/requirements_docs.yml @@ -9,10 +9,10 @@ dependencies: - sphinx~=6.1.0 - sphinx-autodoc-typehints < 2 - sphinx-tabs < 4 + - jsonschema2md >= 1 - ruamel.yaml < 18 - myst-parser < 2 - nbconvert < 8 # To generate HTML files from example notebooks - pip < 24 - pip: - readthedocs-sphinx-search < 0.4 - - jsonschema2md < 1 # available on conda forge but its unnecessary importlib_metadata dependency pin forces sphinx to <0.3.4 diff --git a/setup.py b/setup.py index 478180778..80545c2b9 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def find_calliope_package_data(): setup( name="calliope", - version=__version__, + version=__version__, # noqa: F821 author="Calliope contributors listed in AUTHORS", author_email="stefan@pfenninger.org", description="A multi-scale energy systems modelling framework",