diff --git a/CHANGELOG.md b/CHANGELOG.md index 41417dbb29..054c2fc677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Darts is still in an early development phase, and we cannot always guarantee bac - Removed `IPython` as a dependency. [#1331](https://github.com/unit8co/darts/pull/1331) by [Erik Hasse](https://github.com/erik-hasse) - New models: `DLinearModel` and `NLinearModel` as proposed in [this paper](https://arxiv.org/pdf/2205.13504.pdf). [#1139](https://github.com/unit8co/darts/pull/1139) by [Julien Herzen](https://github.com/hrzn) and [Greg DeVos](https://github.com/gdevos010). - 🔴 Improvements to `TorchForecastingModels`: Load models directly to CPU that were trained on GPU. Save file size reduced. Improved PyTorch Lightning Trainer handling fixing several minor issues. Removed deprecated methods `load_model` and `save_model` [#1371](https://github.com/unit8co/darts/pull/1371) by [Dennis Bader](https://github.com/dennisbader). +- Improvements to encoders: Added support for encoders to all models with covariate support through `add_encoders` at model creation. Encoders now generate the correct minimum required covariate time spans for all models. [#1338](https://github.com/unit8co/darts/pull/1338) by [Dennis Bader](https://github.com/dennisbader). +- Improvements to KalmanForecaster: The model now accepts different TimeSeries for prediction than the ones used to fit the model. [#1338](https://github.com/unit8co/darts/pull/1338) by [Dennis Bader](https://github.com/dennisbader). [Full Changelog](https://github.com/unit8co/darts/compare/0.22.0...master) diff --git a/darts/dataprocessing/encoders/encoder_base.py b/darts/dataprocessing/encoders/encoder_base.py index 189f1e7304..f7f38d3610 100644 --- a/darts/dataprocessing/encoders/encoder_base.py +++ b/darts/dataprocessing/encoders/encoder_base.py @@ -11,7 +11,7 @@ from darts import TimeSeries from darts.dataprocessing.transformers import FittableDataTransformer -from darts.logging import get_logger, raise_if +from darts.logging import get_logger, raise_if, raise_log from darts.utils.timeseries_generation import generate_index SupportedIndex = Union[pd.DatetimeIndex, pd.RangeIndex] @@ -22,33 +22,87 @@ class CovariatesIndexGenerator(ABC): def __init__( self, - input_chunk_length: int, - output_chunk_length: int, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, ): - """ + """:class:`CovariatesIndexGenerator` generates a time index for covariates at training and inference / + prediction time with methods :func:`generate_train_idx()`, and :func:`generate_inference_idx()`. + Without user `covariates`, it generates the minimum required covariate times spans for the corresponding + scenarios described below. With user `covariates`, it simply copies and returns the `covariates` time index. + + It can be used: + A in combination with :class:`LocalForecastingModel`, or in a model agnostic scenario: + All parameters can be ignored. This scenario is only supported by + :class:`FutureCovariatesIndexGenerator`. + B in combination with :class:`RegressionModel`: + Set `input_chunk_length`, `output_chunk_length`, and `lags_covariates`. + `input_chunk_length` is the absolute value of the minimum target lag `abs(min(lags))` used with the + regression model. + Set `output_chunk_length`, and `lags_covariates` with the identical values used at forecasting model + creation. For the covariates lags, use `lags_past_covariates` for class:`PastCovariatesIndexGenerator`, + and `lags_future_covariates` for class:`PastCovariatesIndexGenerator`. + C in combination with :class:`TorchForecastingModel`: + Set `input_chunk_length`, and `output_chunk_length` with the identical values used at forecasting model + creation. + Parameters ---------- input_chunk_length - The length of the emitted past series. + Optionally, the number of input target time steps per chunk. Only required in scenarios B, C. + Corresponds to `input_chunk_length` for :class:`TorchForecastingModel`, or to the absolute minimum target + lag value `abs(min(lags))` for :class:`RegressionModel`. output_chunk_length - The length of the emitted future series. + Optionally, the number of output target time steps per chunk. Only required in scenarios B, and C. + Corresponds to `output_chunk_length` for both :class:`TorchForecastingModel`, and :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers giving the covariates lags used for Darts' RegressionModels. Only required + in scenario B. Corresponds to the lag values from `lags_past_covariates` for past covariates, and + `lags_future_covariates` for future covariates. """ + # check that parameters match one of the scenarios + self._verify_scenario(input_chunk_length, output_chunk_length, lags_covariates) + + # input/output chunk length are guaranteed to both be `None`, or both be defined self.input_chunk_length = input_chunk_length self.output_chunk_length = output_chunk_length + # check lags validity + min_covariates_lag = ( + min(lags_covariates) if lags_covariates is not None else None + ) + max_covariates_lag = ( + max(lags_covariates) if lags_covariates is not None else None + ) + self._verify_lags(min_covariates_lag, max_covariates_lag) + + # from verification min/max lags are guaranteed to either both be None, or both be an integer + if min_covariates_lag is not None: + # we add 1 to the lags so that shift == 0 represents the end of the target series (forecasting point) + shift_start = min_covariates_lag + 1 + shift_end = max_covariates_lag + 1 + else: + shift_start = None + shift_end = None + self.shift_start = shift_start + self.shift_end = shift_end + @abstractmethod def generate_train_idx( self, target: TimeSeries, covariates: Optional[TimeSeries] = None ) -> Tuple[SupportedIndex, pd.Timestamp]: """ - Implement a method that extracts the required covariates index for training. + Generates/extracts time index (or integer index) for covariates at model training time. Parameters ---------- target - The target TimeSeries used during training + The target TimeSeries used during training. covariates - Optionally, the future covariates used for training + Optionally, the covariates used for training. + If given, the returned time index is equal to the `covariates` time index. Else, the returned time index + covers the minimum required covariate time span for training a specific forecasting model. These + requirements are derived from parameters set at :class:`CovariatesIndexGenerator` creation. """ pass @@ -57,16 +111,20 @@ def generate_inference_idx( self, n: int, target: TimeSeries, covariates: Optional[TimeSeries] = None ) -> Tuple[SupportedIndex, pd.Timestamp]: """ - Implement a method that extracts the required covariates index for prediction. + Generates/extracts time index (or integer index) for covariates at model inference / prediction time. Parameters ---------- n - The forecast horizon + The forecasting horizon. target - The target TimeSeries used during training or passed to prediction as `series` + The target TimeSeries used during training or passed to prediction as `series`. covariates - Optionally, the future covariates used for prediction + Optionally, the covariates used for prediction. + If given, the returned time index is equal to the `covariates` time index. Else, the returned time index + covers the minimum required covariate time spans for performing inference / prediction with a specific + forecasting model. These requirements are derived from parameters set at :class:`CovariatesIndexGenerator` + creation. """ pass @@ -79,6 +137,80 @@ def base_component_name(self) -> str: """ pass + def _verify_scenario( + self, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, + ): + # LocalForecastingModel, or model agnostic (only supported by future covariates) + is_scenario_a = ( + isinstance(self, FutureCovariatesIndexGenerator) + and input_chunk_length is None + and output_chunk_length is None + and lags_covariates is None + ) + # RegressionModel + is_scenario_b = ( + input_chunk_length is not None + and output_chunk_length is not None + and lags_covariates is not None + ) + # TorchForecastingModel + is_scenario_c = ( + input_chunk_length is not None + and output_chunk_length is not None + and lags_covariates is None + ) + + if not any([is_scenario_a, is_scenario_b, is_scenario_c]): + raise_log( + ValueError( + "Invalid `CovariatesIndexGenerator` parameter combination: Could not be mapped to an existing " + "scenario, as defined in " + "https://unit8co.github.io/darts/generated_api/darts.dataprocessing.encoders.encoder_base.html" + "#darts.dataprocessing.encoders.encoder_base.CovariatesIndexGenerator" + ), + logger=logger, + ) + + def _verify_lags(self, min_covariates_lag, max_covariates_lag): + """Check the base requirements for `min_covariates_lag` and `max_covariates_lag`: + - both must either be None or an integer + - min_covariates_lag < max_covariates_lag + + This method can be extended by subclasses for past and future covariates lag requirements. + """ + # check that either None one of min/max_covariates_lag are given, or both are given + if (min_covariates_lag is not None and max_covariates_lag is None) or ( + min_covariates_lag is None and max_covariates_lag is not None + ): + raise_log( + ValueError( + "`min_covariates_lag` and `max_covariates_lag` must either both be `None` or both be integers" + ), + logger=logger, + ) + if min_covariates_lag is not None: + # check that if one of the two is given, both must be integers + if not isinstance(min_covariates_lag, int) or not isinstance( + max_covariates_lag, int + ): + raise_log( + ValueError( + "`min_covariates_lag` and `max_covariates_lag` must be both be integers." + ), + logger=logger, + ) + # minimum lag must be less than maximum lag + if min_covariates_lag > max_covariates_lag: + raise_log( + ValueError( + "`min_covariates_lag` must be smaller than/equal to `max_covariates_lag`." + ), + logger=logger, + ) + class PastCovariatesIndexGenerator(CovariatesIndexGenerator): """Generates index for past covariates on train and inference datasets""" @@ -88,41 +220,98 @@ def generate_train_idx( ) -> Tuple[SupportedIndex, pd.Timestamp]: super().generate_train_idx(target, covariates) + + # the returned index depends on the following cases: + # case 0 + # user supplied covariates: simply return the covariate time index; guarantees that an exception is + # raised if user supplied insufficient covariates + # case 1 + # only input_chunk_length and output_chunk_length are given: the complete covariate index is within the + # target index; always True for all models except RegressionModels. + # case 2 + # covariate lags were given (shift_start <= 0 and shift_end <= 0) and + # abs(shift_start - 1) <= input_chunk_length: the complete covariate index is within the target index; + # can only be True for RegressionModels. + # case 3 + # covariate lags were given (shift_start <= 0 and shift_end <= 0) and + # abs(shift_start - 1) > input_chunk_length: we need to add indices before the beginning of the target + # series; can only be True for RegressionModels. + target_end = target.end_time() + if covariates is not None: # case 0 + return covariates.time_index, target_end + + if self.shift_start is None: # case 1 + steps_ahead_start = 0 + else: # case 2 & 3 + steps_ahead_start = self.input_chunk_length + (self.shift_start - 1) + + if not self.shift_end: # case 1 + steps_ahead_end = -self.output_chunk_length + else: # case 2 & 3 + steps_ahead_end = -(self.output_chunk_length - self.shift_end) + + steps_ahead_end = steps_ahead_end if steps_ahead_end else None return ( - covariates.time_index if covariates is not None else target.time_index, + _generate_train_idx(target, steps_ahead_start, steps_ahead_end), target_end, ) def generate_inference_idx( self, n: int, target: TimeSeries, covariates: Optional[TimeSeries] = None ) -> Tuple[SupportedIndex, pd.Timestamp]: - """For prediction (`n` is given) with past covariates we have to distinguish between two cases: - 1) If past covariates are given, we can use them as reference - 2) If past covariates are missing, we need to generate a time index that starts `input_chunk_length` - before the end of `target` and ends `max(0, n - output_chunk_length)` after the end of `target` - """ super().generate_inference_idx(n, target, covariates) + + # for prediction (`n` is given) with past covariates the returned index depends on the following cases: + # case 0 + # user supplied covariates: simply return the covariate time index; guarantees that an exception is + # raised if user supplied insufficient covariates. + # case 1 + # only input_chunk_length and output_chunk_length are given: we need to generate a time index that starts + # `input_chunk_length - 1` before the end of `target` and ends `max(0, n - output_chunk_length)` after the + # end of `target`; always True for all models except RegressionModels. + # case 2 + # covariate lags were given (shift_start <= 0 and shift_end <= 0): we need to generate a time index that + # starts `-shift_start` before the end of `target` and has a length of + # `shift_steps + max(0, n - output_chunk_length)`, where `shift_steps` is the number of time steps between + # `shift_start` and `shift_end`; can only be True for RegressionModels. + target_end = target.end_time() - if covariates is not None: + if covariates is not None: # case 0 return covariates.time_index, target_end - else: - return ( - generate_index( - start=target.end_time() - - target.freq * (self.input_chunk_length - 1), - length=self.input_chunk_length - + max(0, n - self.output_chunk_length), - freq=target.freq, - ), - target_end, - ) + + if self.shift_start is None or self.shift_end is None: # case 1 + steps_back_end = self.input_chunk_length - 1 + n_steps = steps_back_end + 1 + max(0, n - self.output_chunk_length) + else: # case 2 + steps_back_end = -self.shift_start + shift_steps = self.shift_end - self.shift_start + 1 + n_steps = shift_steps + max(0, n - self.output_chunk_length) + + return ( + generate_index( + start=target.end_time() - target.freq * steps_back_end, + length=n_steps, + freq=target.freq, + ), + target_end, + ) @property def base_component_name(self) -> str: return "pc" + def _verify_lags(self, min_covariates_lag, max_covariates_lag): + # general lag checks + super()._verify_lags(min_covariates_lag, max_covariates_lag) + # check past covariate specific lag requirements + if min_covariates_lag is not None and min_covariates_lag >= 0: + raise_log(ValueError("`min_covariates_lag` must be < 0."), logger=logger) + + if max_covariates_lag is not None and max_covariates_lag >= 0: + raise_log(ValueError("`max_covariates_lag` must be < 0."), logger=logger) + class FutureCovariatesIndexGenerator(CovariatesIndexGenerator): """Generates index for future covariates on train and inference datasets.""" @@ -130,39 +319,99 @@ class FutureCovariatesIndexGenerator(CovariatesIndexGenerator): def generate_train_idx( self, target: TimeSeries, covariates: Optional[TimeSeries] = None ) -> Tuple[SupportedIndex, pd.Timestamp]: - """For training (when `n` is `None`) we can simply use the future covariates (if available) or target as - reference to extract the time index. - """ super().generate_train_idx(target, covariates) + + # the returned index depends on the following cases: + # case 0 + # user supplied covariates: simply return the covariate time index; guarantees that models raise an + # exception if user supplied insufficient covariates + # case 1 + # user uses a LocalForecastingModel or model agnostic scenario (input_chunk_length is None): + # simply return the target time index. + # case 2 + # only input_chunk_length and output_chunk_length are given: the complete covariate index is within the + # target index; always True for all models except RegressionModels. + # case 3 + # covariate lags were given and (shift_start <= 0 or shift_end <= 0): historic part of future covariates. + # if shift_end < there will only be the historic part of future covariates. + # If shift_start <= 0 and abs(shift_start - 1) > input_chunk_length: we need to add indices before the + # beginning of the target series; can only be True for RegressionModels. + # case 4 + # covariate lags were given and (shift_start > 0 or shift_end > 0): future part of future covariates. + # if shift_start > 0 there will only be the future part of future covariates. + # If shift_end > 0 and shift_start > input_chunk_length: we need to add indices after the end of the + # target series; can only be True for RegressionModels. + target_end = target.end_time() + + if covariates is not None: # case 0 + return covariates.time_index, target_end + + if self.input_chunk_length is None: # case 1 + return target.time_index, target_end + + if self.shift_start is None: # case 2 + steps_ahead_start = 0 + else: # case 3 + steps_ahead_start = self.input_chunk_length + self.shift_start - 1 + + if self.shift_end is None: # case 2 + steps_ahead_end = 0 + else: # case 4 + steps_ahead_end = -self.output_chunk_length + self.shift_end + steps_ahead_end = steps_ahead_end if steps_ahead_end else None + return ( - covariates.time_index if covariates is not None else target.time_index, + _generate_train_idx(target, steps_ahead_start, steps_ahead_end), target_end, ) def generate_inference_idx( self, n: int, target: TimeSeries, covariates: Optional[TimeSeries] = None ) -> Tuple[SupportedIndex, pd.Timestamp]: - """For prediction (`n` is given) with future covariates we have to distinguish between two cases: - 1) If future covariates are given, we can use them as reference - 2) If future covariates are missing, we need to generate a time index that starts `input_chunk_length` - before the end of `target` and ends `max(n, output_chunk_length)` after the end of `target` - """ + super().generate_inference_idx(n, target, covariates) + + # for prediction (`n` is given) with future covariates the returned index depends on the following cases: + # case 0 + # user supplied covariates: simply return the covariate time index; guarantees that an exception is + # raised if user supplied insufficient covariates + # case 1 + # user uses a LocalForecastingModel or model agnostic scenario (input_chunk_length is None): + # simply return the target time index. + # case 2 + # only input_chunk_length and output_chunk_length are given: we need to generate a time index that starts + # `input_chunk_length - 1` before the end of `target` and ends `max(n, output_chunk_length)` after the + # end of `target`; always True for all models except RegressionModels. + # case 3 + # covariate lags were given: we need to generate a time index that starts `-shift_start` + # steps before the end of `target` and has a length of `shift_steps + max(0, n - output_chunk_length)`, + # where `shift_steps` is `shift_end - shift_start`; can only be True for RegressionModels. + target_end = target.end_time() - if covariates is not None: + if covariates is not None: # case 0 return covariates.time_index, target_end - else: - return ( - generate_index( - start=target.end_time() - - target.freq * (self.input_chunk_length - 1), - length=self.input_chunk_length + max(n, self.output_chunk_length), - freq=target.freq, - ), - target_end, - ) + + if self.input_chunk_length is None: + steps_back_end = -1 + n_steps = n + elif self.shift_start is None: # case 2 + steps_back_end = self.input_chunk_length - 1 + n_steps = steps_back_end + 1 + max(n, self.output_chunk_length) + else: # case 3 + steps_back_end = -self.shift_start + shift_steps = self.shift_end + steps_back_end + 1 + n_steps = shift_steps + max(0, n - self.output_chunk_length) + + return ( + generate_index( + start=target.end_time() - target.freq * steps_back_end, + length=n_steps, + freq=target.freq, + ), + target_end, + ) @property def base_component_name(self) -> str: @@ -529,3 +778,61 @@ def _update_mask(self, covariates: List[TimeSeries]) -> None: def fit_called(self) -> bool: """Return whether or not the transformer has been fitted.""" return self._fit_called + + +def _generate_train_idx(target, steps_ahead_start, steps_ahead_end) -> SupportedIndex: + """The returned index depends on the following cases: + + case 1 + (steps_ahead_start >= 0 and steps_ahead_end is None or <= 1) + the complete index is within the target index; always True for all models except RegressionModels. + case 2 + steps_ahead_start < 0: add indices before the target start time; only possible for RegressionModels + where the minimum past lag is larger than input_chunk_length. + case 3 + steps_ahead_end > 0: add indices after the target end time; only possible for RegressionModels + where the maximum future lag is larger than output_chunk_length. + + Parameters + ---------- + target + the target series. + steps_ahead_start + how many steps ahead of target start time to begin the index. + steps_ahead_end + how many steps ahead of target end time to end the index. + """ + # case 1 + if steps_ahead_start >= 0 and (steps_ahead_end is None or steps_ahead_end <= -1): + return target.time_index[steps_ahead_start:steps_ahead_end] + + # case 2 + idx_start = ( + generate_index( + end=target.start_time() - target.freq, + length=abs(steps_ahead_start), + freq=target.freq, + ) + if steps_ahead_start < 0 + else target.time_index.__class__([]) + ) + + # if `steps_ahead_start >= 0` or `steps_ahead_end <= 0` we must extract a slice of the target series index + center_start = None if steps_ahead_start < 0 else steps_ahead_start + center_end = None if steps_ahead_end > 0 else steps_ahead_end + idx_center = target.time_index[center_start:center_end] + + # case 3 + idx_end = ( + generate_index( + start=target.end_time() + target.freq, + length=abs(steps_ahead_end), + freq=target.freq, + ) + if steps_ahead_end > 0 + else target.time_index.__class__([]) + ) + + # concatenate start, center, and end index + # note: pandas' union() returns type pd.Index(), so we construct index directly from index class + return target.time_index.__class__(idx_start.union(idx_center).union(idx_end)) diff --git a/darts/dataprocessing/encoders/encoders.py b/darts/dataprocessing/encoders/encoders.py index 3ee6956b25..a6176c88d0 100644 --- a/darts/dataprocessing/encoders/encoders.py +++ b/darts/dataprocessing/encoders/encoders.py @@ -155,7 +155,10 @@ from darts.dataprocessing.transformers import FittableDataTransformer from darts.logging import get_logger, raise_if, raise_if_not from darts.timeseries import DIMS -from darts.utils.timeseries_generation import datetime_attribute_timeseries +from darts.utils.timeseries_generation import ( + datetime_attribute_timeseries, + generate_index, +) from darts.utils.utils import seq2series, series2seq SupportedTimeSeries = Union[TimeSeries, Sequence[TimeSeries]] @@ -228,14 +231,16 @@ def base_component_name(self) -> str: class PastCyclicEncoder(CyclicTemporalEncoder): """`CyclicEncoder`: Cyclic encoding of past covariates datetime attributes.""" - def __init__(self, input_chunk_length, output_chunk_length, attribute): + def __init__( + self, + attribute: str, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, + ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute The attribute of the underlying pd.DatetimeIndex from for which to apply cyclic encoding. Must be an attribute of `pd.DatetimeIndex`, or `week` / `weekofyear` / `week_of_year` - e.g. "month", @@ -243,10 +248,26 @@ def __init__(self, input_chunk_length, output_chunk_length, attribute): https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DatetimeIndex.html#pandas.DatetimeIndex. For more information, check out :meth:`datetime_attribute_timeseries() ` + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the past covariate lags. Accepts integer lag values <= -1. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_past_covariates` of :class:`RegressionModel`. """ super().__init__( index_generator=PastCovariatesIndexGenerator( - input_chunk_length, output_chunk_length + input_chunk_length, + output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -255,14 +276,16 @@ def __init__(self, input_chunk_length, output_chunk_length, attribute): class FutureCyclicEncoder(CyclicTemporalEncoder): """`CyclicEncoder`: Cyclic encoding of future covariates datetime attributes.""" - def __init__(self, input_chunk_length, output_chunk_length, attribute): + def __init__( + self, + attribute: str, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, + ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute The attribute of the underlying pd.DatetimeIndex from for which to apply cyclic encoding. Must be an attribute of `pd.DatetimeIndex`, or `week` / `weekofyear` / `week_of_year` - e.g. "month", @@ -270,10 +293,26 @@ def __init__(self, input_chunk_length, output_chunk_length, attribute): https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DatetimeIndex.html#pandas.DatetimeIndex. For more information, check out :meth:`datetime_attribute_timeseries() ` + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the future covariate lags. Accepts all integer values. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_future_covariates` from :class:`RegressionModel`. """ super().__init__( index_generator=FutureCovariatesIndexGenerator( - input_chunk_length, output_chunk_length + input_chunk_length, + output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -331,14 +370,16 @@ def base_component_name(self) -> str: class PastDatetimeAttributeEncoder(DatetimeAttributeEncoder): """Datetime attribute encoder for past covariates.""" - def __init__(self, input_chunk_length, output_chunk_length, attribute): + def __init__( + self, + attribute: str, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, + ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute The attribute of the underlying pd.DatetimeIndex from for which to add scalar information. Must be an attribute of `pd.DatetimeIndex`, or `week` / `weekofyear` / `week_of_year` - e.g. "month", @@ -346,10 +387,26 @@ def __init__(self, input_chunk_length, output_chunk_length, attribute): https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DatetimeIndex.html#pandas.DatetimeIndex. For more information, check out :meth:`datetime_attribute_timeseries() ` + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the past covariate lags. Accepts integer lag values <= -1. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_past_covariates` of :class:`RegressionModel`. """ super().__init__( index_generator=PastCovariatesIndexGenerator( - input_chunk_length, output_chunk_length + input_chunk_length, + output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -358,14 +415,16 @@ def __init__(self, input_chunk_length, output_chunk_length, attribute): class FutureDatetimeAttributeEncoder(DatetimeAttributeEncoder): """Datetime attribute encoder for future covariates.""" - def __init__(self, input_chunk_length, output_chunk_length, attribute): + def __init__( + self, + attribute: str, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, + ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute The attribute of the underlying pd.DatetimeIndex from for which to add scalar information. Must be an attribute of `pd.DatetimeIndex`, or `week` / `weekofyear` / `week_of_year` - e.g. "month", @@ -373,10 +432,26 @@ def __init__(self, input_chunk_length, output_chunk_length, attribute): https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DatetimeIndex.html#pandas.DatetimeIndex. For more information, check out :meth:`datetime_attribute_timeseries() ` + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the future covariate lags. Accepts all integer values. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_future_covariates` from :class:`RegressionModel`. """ super().__init__( index_generator=FutureCovariatesIndexGenerator( - input_chunk_length, output_chunk_length + input_chunk_length, + output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -414,11 +489,28 @@ def _encode( """Applies cyclic encoding from `datetime_attribute_timeseries()` to `self.attribute` of `index`. For attribute=='relative', the reference point/index is the prediction/forecast index of the target series. """ - idx_larger_end = (index <= target_end).sum() - 1 super()._encode(index, target_end, dtype) + + idx_larger_end = (index <= target_end).sum() + freq = index.freq if isinstance(index, pd.DatetimeIndex) else index.step + if idx_larger_end: + idx_larger_end -= 1 + if index[0] > target_end: + idx_diff = ( + len(generate_index(start=target_end, end=index[0], freq=freq)) - 1 + ) + elif index[-1] < target_end: + idx_diff = ( + -len(generate_index(start=index[-1], end=target_end, freq=freq)) + 1 + ) + else: + idx_diff = 0 return TimeSeries.from_times_and_values( times=index, - values=np.arange(-idx_larger_end, -idx_larger_end + len(index)), + values=np.arange( + start=idx_diff - idx_larger_end, + stop=idx_diff - idx_larger_end + len(index), + ), columns=[self.base_component_name + self.attribute], ).astype(np.dtype(dtype)) @@ -444,23 +536,38 @@ class PastIntegerIndexEncoder(IntegerIndexEncoder): """ def __init__( - self, input_chunk_length: int, output_chunk_length: int, attribute: str + self, + attribute: str, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute Currently only 'relative' is supported. The generated encoded values will range from (-inf, inf) and the target series end time will be used as a reference to evaluate the relative index positions. + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the past covariate lags. Accepts integer lag values <= -1. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_past_covariates` of :class:`RegressionModel`. """ super().__init__( index_generator=PastCovariatesIndexGenerator( input_chunk_length, output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -472,23 +579,38 @@ class FutureIntegerIndexEncoder(IntegerIndexEncoder): """ def __init__( - self, input_chunk_length: int, output_chunk_length: int, attribute: str + self, + attribute: str, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute Currently only 'relative' is supported. The generated encoded values will range from (-inf, inf) and the target series end time will be used as a reference to evaluate the relative index positions. + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the future covariate lags. Accepts all integer values. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_future_covariates` from :class:`RegressionModel`. """ super().__init__( index_generator=FutureCovariatesIndexGenerator( input_chunk_length, output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -557,27 +679,40 @@ class PastCallableIndexEncoder(CallableIndexEncoder): def __init__( self, - input_chunk_length: int, - output_chunk_length: int, - attribute: Union[str, Callable], + attribute: Callable, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute A callable that takes an index `index` of type `(pd.DatetimeIndex, pd.RangeIndex)` as input and returns a np.ndarray of shape `(len(index),)`. An example for a correct `attribute` for `index` of type pd.DatetimeIndex: ``attribute = lambda index: (index.year - 1950) / 50``. And for pd.RangeIndex: ``attribute = lambda index: (index - 1950) / 50`` + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the past covariate lags. Accepts integer lag values <= -1. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_past_covariates` of :class:`RegressionModel`. """ super().__init__( index_generator=PastCovariatesIndexGenerator( - input_chunk_length, output_chunk_length + input_chunk_length, + output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -590,28 +725,40 @@ class FutureCallableIndexEncoder(CallableIndexEncoder): def __init__( self, - input_chunk_length: int, - output_chunk_length: int, - attribute: Union[str, Callable], + attribute: Callable, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_covariates: Optional[List[int]] = None, ): """ Parameters ---------- - input_chunk_length - The length of the emitted past series. - output_chunk_length - The length of the emitted future series. attribute A callable that takes an index `index` of type `(pd.DatetimeIndex, pd.RangeIndex)` as input and returns a np.ndarray of shape `(len(index),)`. An example for a correct `attribute` for `index` of type pd.DatetimeIndex: ``attribute = lambda index: (index.year - 1950) / 50``. And for pd.RangeIndex: ``attribute = lambda index: (index - 1950) / 50`` + input_chunk_length + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. + output_chunk_length + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_covariates + Optionally, a list of integers representing the future covariate lags. Accepts all integer values. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_future_covariates` from :class:`RegressionModel`. """ - super().__init__( index_generator=FutureCovariatesIndexGenerator( - input_chunk_length, output_chunk_length + input_chunk_length, + output_chunk_length, + lags_covariates=lags_covariates, ), attribute=attribute, ) @@ -625,18 +772,17 @@ class SequentialEncoder(Encoder): def __init__( self, add_encoders: Dict, - input_chunk_length: int, - output_chunk_length: int, + input_chunk_length: Optional[int] = None, + output_chunk_length: Optional[int] = None, + lags_past_covariates: Optional[List[int]] = None, + lags_future_covariates: Optional[List[int]] = None, takes_past_covariates: bool = False, takes_future_covariates: bool = False, ) -> None: - """ - SequentialEncoder automatically creates encoder objects from parameter `add_encoders` used when creating a - `TorchForecastingModel`. - - * Only kwarg `add_encoders` of type `Optional[Dict]` will be used to extract the encoders. - For example: `model = MyModel(..., add_encoders={...}, ...)` + SequentialEncoder automatically creates encoder objects from parameter `add_encoders`. `add_encoders` can also + be set directly in all of Darts' `ForecastingModels`. This will automatically set up a + :class:`SequentialEncoder` tailored to the settings of the underlying forecasting model. The `add_encoders` dict must follow this convention: `{encoder keyword: {temporal keyword: List[attributes]}, ..., transformer keyword: transformer object}` @@ -691,17 +837,30 @@ def __init__( Parameters ---------- add_encoders - The parameters used at `TorchForecastingModel` model creation. + A dictionary with the encoder settings. input_chunk_length - The length of the emitted past series. + Optionally, the number of input target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `input_chunk_length` from :class:`TorchForecastingModel`, or to the absolute + minimum target lag value `abs(min(lags))` for :class:`RegressionModel`. output_chunk_length - The length of the emitted future series. + Optionally, the number of output target time steps per chunk. Only required for + :class:`TorchForecastingModel`, and :class:`RegressionModel`. + Corresponds to parameter `output_chunk_length` from both :class:`TorchForecastingModel`, and + :class:`RegressionModel`. + lags_past_covariates + Optionally, a list of integers representing the past covariate lags. Accepts integer lag values <= -1. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_past_covariates` of :class:`RegressionModel`. + lags_future_covariates + Optionally, a list of integers representing the future covariate lags. Accepts all integer values. + Only required for :class:`RegressionModel`. + Corresponds to the lag values from parameter `lags_future_covariates` from :class:`RegressionModel`. takes_past_covariates - Whether or not the `TrainingDataset` takes past covariates + Whether to encode/generate past covariates. takes_future_covariates - Whether or not the `TrainingDataset` takes past covariates + Whether to encode/generate future covariates. """ - super().__init__() self.params = add_encoders self.input_chunk_length = input_chunk_length @@ -709,6 +868,8 @@ def __init__( self.encoding_available = False self.takes_past_covariates = takes_past_covariates self.takes_future_covariates = takes_future_covariates + self.lags_past_covariates = lags_past_covariates + self.lags_future_covariates = lags_future_covariates # encoders self._past_encoders: List[SingleEncoder] = [] @@ -746,9 +907,9 @@ def encode_train( future_covariates Optionally, the future covariates used for training. encode_past - Whether or not to apply encoders for past covariates + Whether to apply encoders for past covariates encode_future - Whether or not to apply encoders for future covariates + Whether to apply encoders for future covariates Returns ------- Tuple[past_covariates, future_covariates] @@ -810,9 +971,9 @@ def encode_inference( future_covariates Optionally, the future covariates used for training. encode_past - Whether or not to apply encoders for past covariates + Whether to apply encoders for past covariates encode_future - Whether or not to apply encoders for future covariates + Whether to apply encoders for future covariates Returns ------- @@ -850,7 +1011,6 @@ def _launch_encoder( If `n` is not `None` it is a prediction, otherwise it is training. """ - if not self.encoding_available: return past_covariates, future_covariates @@ -912,22 +1072,23 @@ def _encode_sequence( [covariates] if isinstance(covariates, TimeSeries) else covariates ) - for ts, pc in zip(target, covariates): + for ts, covs in zip(target, covariates): # drop encoder components if they are in input covariates - pc = self._drop_encoded_components( - covariates=pc, components=getattr(self, f"{covariates_type}_components") + covs = self._drop_encoded_components( + covariates=covs, + components=getattr(self, f"{covariates_type}_components"), ) encoded = concatenate( [ getattr(enc, encode_method)( - target=ts, covariates=pc, merge_covariates=False, n=n + target=ts, covariates=covs, merge_covariates=False, n=n ) for enc in encoders ], axis=DIMS[1], ) encoded_sequence.append( - self._merge_covariates(encoded=encoded, covariates=pc) + self._merge_covariates(encoded=encoded, covariates=covs) ) if transformer is not None: @@ -1029,13 +1190,19 @@ def _setup_encoders(self, params: Dict) -> None: self._past_encoders = [ self.encoder_map[enc_id]( - self.input_chunk_length, self.output_chunk_length, attr + attribute=attr, + input_chunk_length=self.input_chunk_length, + output_chunk_length=self.output_chunk_length, + lags_covariates=self.lags_past_covariates, ) for enc_id, attr in past_encoders ] self._future_encoders = [ self.encoder_map[enc_id]( - self.input_chunk_length, self.output_chunk_length, attr + attribute=attr, + input_chunk_length=self.input_chunk_length, + output_chunk_length=self.output_chunk_length, + lags_covariates=self.lags_future_covariates, ) for enc_id, attr in future_encoders ] @@ -1091,7 +1258,6 @@ def _process_input_encoders(self, params: Dict) -> Tuple[List, List]: 1) if the outermost key is other than (`past`, `future`) 2) if the innermost values are other than type `str` or `Sequence` """ - if not params: return [], [] @@ -1169,7 +1335,6 @@ def _process_input_transformer( Dict from parameter `add_encoders` (kwargs) used at model creation. Relevant parameters are: * params={'transformer': Scaler()} """ - if not params: return None, [], [] diff --git a/darts/explainability/explainability.py b/darts/explainability/explainability.py index 88eefcbedf..39bd185b72 100644 --- a/darts/explainability/explainability.py +++ b/darts/explainability/explainability.py @@ -99,8 +99,7 @@ def __init__( ( background_past_covariates, background_future_covariates, - ) = self.model.generate_predict_encodings( - n=len(background_series) - self.model.min_train_series_length, + ) = self.model.generate_fit_encodings( series=background_series, past_covariates=background_past_covariates, future_covariates=background_future_covariates, diff --git a/darts/explainability/shap_explainer.py b/darts/explainability/shap_explainer.py index 4e993590d5..a2c9517d38 100644 --- a/darts/explainability/shap_explainer.py +++ b/darts/explainability/shap_explainer.py @@ -201,22 +201,12 @@ def explain( foreground_past_covariates = series2seq(foreground_past_covariates) foreground_future_covariates = series2seq(foreground_future_covariates) - # For encoding inference, in case we don't have past and future covariates, - # We need to provide the minimum elements for the foreground_series. If the foreground_series - # is a list of TimeSeries, we need to provide the minimum elements for each TimeSeries. - foreground_series_start_encoder = [] - for i in range(len(foreground_series)): - foreground_series_start_encoder.append( - foreground_series[i][: self.model.min_train_series_length] - ) - if self.model.encoders.encoding_available: ( foreground_past_covariates, foreground_future_covariates, - ) = self.model.generate_predict_encodings( - n=len(foreground_series) - self.model.min_train_series_length, - series=foreground_series_start_encoder, + ) = self.model.generate_fit_encodings( + series=foreground_series, past_covariates=foreground_past_covariates, future_covariates=foreground_future_covariates, ) @@ -377,8 +367,7 @@ def force_plot_from_ts( ( foreground_past_covariates, foreground_future_covariates, - ) = self.model.generate_predict_encodings( - n=len(foreground_series) - self.model.min_train_series_length, + ) = self.model.generate_fit_encodings( series=foreground_series, past_covariates=foreground_past_covariates, future_covariates=foreground_future_covariates, diff --git a/darts/models/filtering/filtering_model.py b/darts/models/filtering/filtering_model.py index 4328dc1d9e..98ae7c838f 100644 --- a/darts/models/filtering/filtering_model.py +++ b/darts/models/filtering/filtering_model.py @@ -20,6 +20,7 @@ class FilteringModel(ABC): @abstractmethod def __init__(self): + self._expect_covariates = False pass @abstractmethod diff --git a/darts/models/filtering/kalman_filter.py b/darts/models/filtering/kalman_filter.py index ff4a61acf3..a390122d5f 100644 --- a/darts/models/filtering/kalman_filter.py +++ b/darts/models/filtering/kalman_filter.py @@ -55,7 +55,6 @@ def __init__(self, dim_x: int = 1, kf: Optional[Kalman] = None): # TODO: Add support for x_init. Needs reimplementation of NFourSID. super().__init__() - self._expect_covariates = False if kf is None: self.kf = None diff --git a/darts/models/forecasting/arima.py b/darts/models/forecasting/arima.py index 839f09e537..3175bceabe 100644 --- a/darts/models/forecasting/arima.py +++ b/darts/models/forecasting/arima.py @@ -33,6 +33,7 @@ def __init__( seasonal_order: Tuple[int, int, int, int] = (0, 0, 0, 0), trend: Optional[str] = None, random_state: int = 0, + add_encoders: Optional[dict] = None, ): """ARIMA ARIMA-type models extensible with exogenous variables (future covariates) @@ -54,8 +55,28 @@ def __init__( Parameter controlling the deterministic trend. 'n' indicates no trend, 'c' a constant term, 't' linear trend in time, and 'ct' includes both. Default is 'c' for models without integration, and no trend for models with integration. + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. """ - super().__init__() + super().__init__(add_encoders=add_encoders) self.order = p, d, q self.seasonal_order = seasonal_order self.trend = trend diff --git a/darts/models/forecasting/auto_arima.py b/darts/models/forecasting/auto_arima.py index d4e096e3c9..d77847eec4 100644 --- a/darts/models/forecasting/auto_arima.py +++ b/darts/models/forecasting/auto_arima.py @@ -17,7 +17,9 @@ class AutoARIMA(FutureCovariatesLocalForecastingModel): - def __init__(self, *autoarima_args, **autoarima_kwargs): + def __init__( + self, *autoarima_args, add_encoders: Optional[dict] = None, **autoarima_kwargs + ): """Auto-ARIMA This implementation is a thin wrapper around `pmdarima AutoARIMA model @@ -40,8 +42,28 @@ def __init__(self, *autoarima_args, **autoarima_kwargs): Positional arguments for the pmdarima.AutoARIMA model autoarima_kwargs Keyword arguments for the pmdarima.AutoARIMA model + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. """ - super().__init__() + super().__init__(add_encoders=add_encoders) self.model = PmdAutoARIMA(*autoarima_args, **autoarima_kwargs) self.trend = self.model.trend diff --git a/darts/models/forecasting/croston.py b/darts/models/forecasting/croston.py index f2a105bd91..452fa760ae 100644 --- a/darts/models/forecasting/croston.py +++ b/darts/models/forecasting/croston.py @@ -17,7 +17,11 @@ class Croston(FutureCovariatesLocalForecastingModel): def __init__( - self, version: str = "classic", alpha_d: float = None, alpha_p: float = None + self, + version: str = "classic", + alpha_d: float = None, + alpha_p: float = None, + add_encoders: Optional[dict] = None, ): """An implementation of the `Croston method `_ for intermittent @@ -42,6 +46,26 @@ def __init__( For the "tsb" version, the alpha smoothing parameter to apply on demand. alpha_p For the "tsb" version, the alpha smoothing parameter to apply on probability. + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. References ---------- @@ -51,7 +75,7 @@ def __init__( Intermittent demand: Linking forecasting to inventory obsolescence. European Journal of Operational Research, 214(3):606 – 615, 2011. """ - super().__init__() + super().__init__(add_encoders=add_encoders) raise_if_not( version.lower() in ["classic", "optimized", "sba", "tsb"], 'The provided "version" parameter must be set to "classic", "optimized", "sba" or "tsb".', diff --git a/darts/models/forecasting/forecasting_model.py b/darts/models/forecasting/forecasting_model.py index a600e28395..a4448b0d9a 100644 --- a/darts/models/forecasting/forecasting_model.py +++ b/darts/models/forecasting/forecasting_model.py @@ -115,7 +115,6 @@ def __init__(self, *args, **kwargs): self.static_covariates: Optional[pd.DataFrame] = None self._expect_past_covariates, self._expect_future_covariates = False, False - self._uses_past_covariates, self._uses_future_covariates = False, False # state; whether the model has been fit (on a single time series) @@ -124,6 +123,18 @@ def __init__(self, *args, **kwargs): # extract and store sub class model creation parameters self._model_params = self._extract_model_creation_params() + if "add_encoders" not in kwargs: + raise_log( + NotImplementedError( + "Model subclass must pass the `add_encoders` parameter to base class." + ), + logger=logger, + ) + + # by default models do not use encoders + self.add_encoders = kwargs["add_encoders"] + self.encoders: Optional[SequentialEncoder] = None + @abstractmethod def fit(self, series: TimeSeries) -> "ForecastingModel": """Fit/train the model on the provided series. @@ -1366,6 +1377,133 @@ def residuals( return residuals_list if len(residuals_list) > 1 else residuals_list[0] + def initialize_encoders(self) -> SequentialEncoder: + """instantiates the SequentialEncoder object based on self._model_encoder_settings and parameter + ``add_encoders`` used at model creation""" + ( + input_chunk_length, + output_chunk_length, + takes_past_covariates, + takes_future_covariates, + lags_past_covariates, + lags_future_covariates, + ) = self._model_encoder_settings + + return SequentialEncoder( + add_encoders=self.add_encoders, + input_chunk_length=input_chunk_length, + output_chunk_length=output_chunk_length, + lags_past_covariates=lags_past_covariates, + lags_future_covariates=lags_future_covariates, + takes_past_covariates=takes_past_covariates, + takes_future_covariates=takes_future_covariates, + ) + + def generate_fit_encodings( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + ) -> Tuple[ + Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]] + ]: + """Generates the covariate encodings that were used/generated for fitting the model and returns a tuple of + past, and future covariates series with the original and encoded covariates stacked together. The encodings are + generated by the encoders defined at model creation with parameter `add_encoders`. Pass the same `series`, + `past_covariates`, and `future_covariates` that you used to train/fit the model. + + Parameters + ---------- + series + The series or sequence of series with the target values used when fitting the model. + past_covariates + Optionally, the series or sequence of series with the past-observed covariates used when fitting the model. + future_covariates + Optionally, the series or sequence of series with the future-known covariates used when fitting the model. + + Returns + ------- + Tuple[Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]]] + A tuple of (past covariates, future covariates). Each covariate contains the original as well as the + encoded covariates. + """ + raise_if( + self.encoders is None or not self.encoders.encoding_available, + "Encodings are not available. Consider adding parameter `add_encoders` at model creation and fitting the " + "model with `model.fit()` before.", + logger=logger, + ) + return self.encoders.encode_train( + target=series, + past_covariates=past_covariates, + future_covariates=future_covariates, + ) + + def generate_predict_encodings( + self, + n: int, + series: Union[TimeSeries, Sequence[TimeSeries]], + past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + ) -> Tuple[ + Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]] + ]: + """Generates covariate encodings for the inference/prediction set and returns a tuple of past, and future + covariates series with the original and encoded covariates stacked together. The encodings are generated by the + encoders defined at model creation with parameter `add_encoders`. Pass the same `series`, `past_covariates`, + and `future_covariates` that you intend to use for prediction. + + Parameters + ---------- + n + The number of prediction time steps after the end of `series` intended to be used for prediction. + series + The series or sequence of series with target values intended to be used for prediction. + past_covariates + Optionally, the past-observed covariates series intended to be used for prediction. The dimensions must + match those of the covariates used for training. + future_covariates + Optionally, the future-known covariates series intended to be used for prediction. The dimensions must + match those of the covariates used for training. + + Returns + ------- + Tuple[Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]]] + A tuple of (past covariates, future covariates). Each covariate contains the original as well as the + encoded covariates. + """ + raise_if( + self.encoders is None or not self.encoders.encoding_available, + "Encodings are not available. Consider adding parameter `add_encoders` at model creation and fitting the " + "model with `model.fit()` before.", + logger=logger, + ) + return self.encoders.encode_inference( + n=n, + target=series, + past_covariates=past_covariates, + future_covariates=future_covariates, + ) + + @property + @abstractmethod + def _model_encoder_settings( + self, + ) -> Tuple[ + Optional[int], + Optional[int], + bool, + bool, + Optional[List[int]], + Optional[List[int]], + ]: + """Abstract property that returns model specific encoder settings that are used to initialize the encoders. + + Must return Tuple (input_chunk_length, output_chunk_length, takes_past_covariates, takes_future_covariates, + lags_past_covariates, lags_future_covariates). + """ + pass + @classmethod def _sample_params(model_class, params, n_random_samples): """Select the absolute number of samples randomly if an integer has been supplied. If a float has been @@ -1488,6 +1626,22 @@ class LocalForecastingModel(ForecastingModel, ABC): All implementations must implement the `fit()` and `predict()` methods. """ + def __init__(self, add_encoders: Optional[dict] = None): + super().__init__(add_encoders=add_encoders) + + @property + def _model_encoder_settings( + self, + ) -> Tuple[ + Optional[int], + Optional[int], + bool, + bool, + Optional[List[int]], + Optional[List[int]], + ]: + return None, None, False, False, None, None + @abstractmethod def fit(self, series: TimeSeries) -> "LocalForecastingModel": super().fit(series) @@ -1516,11 +1670,7 @@ class GlobalForecastingModel(ForecastingModel, ABC): """ def __init__(self, add_encoders: Optional[dict] = None): - super().__init__() - - # by default models do not use encoders - self.add_encoders = add_encoders - self.encoders: Optional[SequentialEncoder] = None + super().__init__(add_encoders=add_encoders) @abstractmethod def fit( @@ -1684,125 +1834,6 @@ def _supports_non_retrainable_historical_forecasts(self) -> bool: """GlobalForecastingModel supports historical forecasts without retraining the model""" return True - @property - @abstractmethod - def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: - """Abstract property that returns model specific encoder settings that are used to initialize the encoders. - - Must return Tuple (input_chunk_length, output_chunk_length, takes_past_covariates, takes_future_covariates) - """ - pass - - def initialize_encoders(self) -> SequentialEncoder: - """instantiates the SequentialEncoder object based on self._model_encoder_settings and parameter - ``add_encoders`` used at model creation""" - ( - input_chunk_length, - output_chunk_length, - takes_past_covariates, - takes_future_covariates, - ) = self._model_encoder_settings - - return SequentialEncoder( - add_encoders=self.add_encoders, - input_chunk_length=input_chunk_length, - output_chunk_length=output_chunk_length, - takes_past_covariates=takes_past_covariates, - takes_future_covariates=takes_future_covariates, - ) - - def generate_fit_encodings( - self, - series: Union[TimeSeries, Sequence[TimeSeries]], - past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, - future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, - ) -> Tuple[ - Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]] - ]: - """Generates the covariate encodings that were used/generated for fitting the model and returns a tuple of - past, and future covariates series with the original and encoded covariates stacked together. The encodings are - generated by the encoders defined at model creation with parameter `add_encoders`. Pass the same `series`, - `past_covariates`, and `future_covariates` that you used to train/fit the model. - - Parameters - ---------- - series - The series or sequence of series with the target values used when fitting the model. - past_covariates - Optionally, the series or sequence of series with the past-observed covariates used when fitting the model. - future_covariates - Optionally, the series or sequence of series with the future-known covariates used when fitting the model. - - Returns - ------- - Tuple[Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]]] - A tuple of (past covariates, future covariates). Each covariate contains the original as well as the - encoded covariates. - """ - raise_if( - self.encoders is None or not self.encoders.encoding_available, - "Encodings are not available. Consider adding parameter `add_encoders` at model creation and fitting the " - "model with `model.fit()` before.", - logger=logger, - ) - return self.encoders.encode_train( - target=series, - past_covariates=past_covariates, - future_covariates=future_covariates, - ) - - def generate_predict_encodings( - self, - n: int, - series: Union[TimeSeries, Sequence[TimeSeries]], - past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, - future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, - ) -> Tuple[ - Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]] - ]: - """Generates covariate encodings for the inference/prediction set and returns a tuple of past, and future - covariates series with the original and encoded covariates stacked together. The encodings are generated by the - encoders defined at model creation with parameter `add_encoders`. Pass the same `series`, `past_covariates`, - and `future_covariates` that you intend to use for prediction. - - Parameters - ---------- - n - The number of prediction time steps after the end of `series` intended to be used for prediction. - series - The series or sequence of series with target values intended to be used for prediction. - past_covariates - Optionally, the past-observed covariates series intended to be used for prediction. The dimensions must - match those of the covariates used for training. - future_covariates - Optionally, the future-known covariates series intended to be used for prediction. The dimensions must - match those of the covariates used for training. - - Returns - ------- - Tuple[Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]]] - A tuple of (past covariates, future covariates). Each covariate contains the original as well as the - encoded covariates. - """ - raise_if( - self.encoders is None or not self.encoders.encoding_available, - "Encodings are not available. Consider adding parameter `add_encoders` at model creation and fitting the " - "model with `model.fit()` before.", - logger=logger, - ) - return self.encoders.encode_inference( - n=self._get_encoders_n(n), - target=series, - past_covariates=past_covariates, - future_covariates=future_covariates, - ) - - def _get_encoders_n(self, n) -> int: - """Returns the number of prediction steps for generating with `model.encoders.generate_predict_encodings()`. - Subclasses can have different requirements for setting `n`. The most general case simply returns `n` as is. - """ - return n - class FutureCovariatesLocalForecastingModel(LocalForecastingModel, ABC): """The base class for future covariates "local" forecasting models, handling single uni- or multivariate target @@ -1847,10 +1878,18 @@ def fit(self, series: TimeSeries, future_covariates: Optional[TimeSeries] = None series.has_same_time_as(future_covariates), "The provided `future_covariates` series must contain at least the same time steps/" "indices as the target `series`.", - logger, + logger=logger, ) self._expect_future_covariates = True + self.encoders = self.initialize_encoders() + if self.encoders.encoding_available: + _, future_covariates = self.generate_fit_encodings( + series=series, + past_covariates=None, + future_covariates=future_covariates, + ) + super().fit(series) return self._fit(series, future_covariates=future_covariates) @@ -1892,19 +1931,16 @@ def predict( super().predict(n, num_samples) - if self._expect_future_covariates and future_covariates is None: - raise_log( - ValueError( - "The model has been trained with `future_covariates` variable. Some matching " - "`future_covariates` variables have to be provided to `predict()`." + # avoid generating encodings again if subclass has already generated them + if not self._supress_generate_predict_encoding: + self._verify_passed_predict_covariates(future_covariates) + if self.encoders is not None and self.encoders.encoding_available: + _, future_covariates = self.generate_predict_encodings( + n=n, + series=self.training_series, + past_covariates=None, + future_covariates=future_covariates, ) - ) - - raise_if( - not self._expect_future_covariates and future_covariates is not None, - "The model has been trained without `future_covariates` variable, but the " - "`future_covariates` parameter provided to `predict()` is not None.", - ) if future_covariates is not None: start = self.training_series.end_time() + self.training_series.freq @@ -1925,9 +1961,9 @@ def predict( if isinstance(future_covariates.time_index, pd.DatetimeIndex) else n ) - future_covariates = future_covariates[ - start : start + offset * self.training_series.freq - ] + future_covariates = future_covariates.slice( + start, start + offset * self.training_series.freq + ) raise_if_not( len(future_covariates) == n, @@ -1977,6 +2013,41 @@ def _predict_wrapper( verbose=verbose, ) + @property + def _model_encoder_settings( + self, + ) -> Tuple[ + Optional[int], + Optional[int], + bool, + bool, + Optional[List[int]], + Optional[List[int]], + ]: + return None, None, False, True, None, None + + def _verify_passed_predict_covariates(self, future_covariates): + """Simple check if user supplied/did not supply covariates as done at fitting time.""" + if self._expect_future_covariates and future_covariates is None: + raise_log( + ValueError( + "The model has been trained with `future_covariates` variable. Some matching " + "`future_covariates` variables have to be provided to `predict()`." + ) + ) + if not self._expect_future_covariates and future_covariates is not None: + raise_log( + ValueError( + "The model has been trained without `future_covariates` variable, but the " + "`future_covariates` parameter provided to `predict()` is not None.", + ) + ) + + @property + def _supress_generate_predict_encoding(self) -> bool: + """Controls wether encodings should be generated in :func:`FutureCovariatesLocalForecastingModel.predict()``""" + return False + class TransferableFutureCovariatesLocalForecastingModel( FutureCovariatesLocalForecastingModel, ABC @@ -2032,13 +2103,13 @@ def predict( ------- TimeSeries, a single time series containing the `n` next points after then end of the training series. """ - - if self._expect_future_covariates and future_covariates is None: - raise_log( - ValueError( - "The model has been trained with `future_covariates` variable. Some matching " - "`future_covariates` variables have to be provided to `predict()`." - ) + self._verify_passed_predict_covariates(future_covariates) + if self.encoders is not None and self.encoders.encoding_available: + _, future_covariates = self.generate_predict_encodings( + n=n, + series=series if series is not None else self.training_series, + past_covariates=None, + future_covariates=future_covariates, ) historic_future_covariates = None @@ -2084,6 +2155,38 @@ def predict( return result + def generate_predict_encodings( + self, + n: int, + series: Union[TimeSeries, Sequence[TimeSeries]], + past_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]] = None, + ) -> Tuple[ + Union[TimeSeries, Sequence[TimeSeries]], Union[TimeSeries, Sequence[TimeSeries]] + ]: + raise_if( + self.encoders is None or not self.encoders.encoding_available, + "Encodings are not available. Consider adding parameter `add_encoders` at model creation and fitting the " + "model with `model.fit()` before.", + logger=logger, + ) + _, future_covariates_future = self.encoders.encode_inference( + n=n, + target=series, + past_covariates=past_covariates, + future_covariates=future_covariates, + ) + + if future_covariates is not None: + return _, future_covariates_future + + _, future_covariates_historic = self.encoders.encode_train( + target=series, + past_covariates=past_covariates, + future_covariates=future_covariates, + ) + return _, future_covariates_historic.append(future_covariates_future) + @abstractmethod def _predict( self, @@ -2119,6 +2222,10 @@ def _predict_wrapper( def _supports_non_retrainable_historical_forecasts(self) -> bool: return True + @property + def _supress_generate_predict_encoding(self) -> bool: + return True + @property def extreme_lags(self): return (-1, 1, None, 0, 0) diff --git a/darts/models/forecasting/kalman_forecaster.py b/darts/models/forecasting/kalman_forecaster.py index 9dc98598fc..cfc9264275 100644 --- a/darts/models/forecasting/kalman_forecaster.py +++ b/darts/models/forecasting/kalman_forecaster.py @@ -18,15 +18,20 @@ from darts.logging import get_logger from darts.models.filtering.kalman_filter import KalmanFilter from darts.models.forecasting.forecasting_model import ( - FutureCovariatesLocalForecastingModel, + TransferableFutureCovariatesLocalForecastingModel, ) from darts.timeseries import TimeSeries logger = get_logger(__name__) -class KalmanForecaster(FutureCovariatesLocalForecastingModel): - def __init__(self, dim_x: int = 1, kf: Optional[Kalman] = None): +class KalmanForecaster(TransferableFutureCovariatesLocalForecastingModel): + def __init__( + self, + dim_x: int = 1, + kf: Optional[Kalman] = None, + add_encoders: Optional[dict] = None, + ): """Kalman filter Forecaster This model uses a Kalman filter to produce forecasts. It uses a @@ -50,8 +55,28 @@ def __init__(self, dim_x: int = 1, kf: Optional[Kalman] = None): calling `predict()`. If this is specified, it is still necessary to call `fit()` before calling `predict()`, although this will have no effect on the Kalman filter. + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. """ - super().__init__() + super().__init__(add_encoders=add_encoders) self.dim_x = dim_x self.kf = kf self.darts_kf = KalmanFilter(dim_x, kf) @@ -67,17 +92,33 @@ def _fit(self, series: TimeSeries, future_covariates: Optional[TimeSeries] = Non return self + def predict( + self, + n: int, + series: Optional[TimeSeries] = None, + future_covariates: Optional[TimeSeries] = None, + num_samples: int = 1, + **kwargs, + ) -> TimeSeries: + # we override `predict()` to pass a non-None `series`, so that historic_future_covariates + # will be passed to `_predict()` + series = series if series is not None else self.training_series + return super().predict(n, series, future_covariates, num_samples, **kwargs) + def _predict( self, n: int, + series: Optional[TimeSeries] = None, + historic_future_covariates: Optional[TimeSeries] = None, future_covariates: Optional[TimeSeries] = None, num_samples: int = 1, verbose: bool = False, ) -> TimeSeries: - super()._predict(n, future_covariates, num_samples) - - time_index = self._generate_new_dates(n) + super()._predict( + n, series, historic_future_covariates, future_covariates, num_samples + ) + time_index = self._generate_new_dates(n, input_series=series) placeholder_vals = np.zeros((n, self.training_series.width)) * np.nan series_future = TimeSeries.from_times_and_values( time_index, @@ -86,9 +127,13 @@ def _predict( static_covariates=self.training_series.static_covariates, hierarchy=self.training_series.hierarchy, ) - whole_series = self.training_series.append(series_future) + + series = series.append(series_future) + if historic_future_covariates is not None: + future_covariates = historic_future_covariates.append(future_covariates) + filtered_series = self.darts_kf.filter( - whole_series, covariates=future_covariates, num_samples=num_samples + series=series, covariates=future_covariates, num_samples=num_samples ) return filtered_series[-n:] diff --git a/darts/models/forecasting/prophet_model.py b/darts/models/forecasting/prophet_model.py index 9fef1e9319..1927faaf77 100644 --- a/darts/models/forecasting/prophet_model.py +++ b/darts/models/forecasting/prophet_model.py @@ -27,6 +27,7 @@ def __init__( add_seasonalities: Optional[Union[dict, List[dict]]] = None, country_holidays: Optional[str] = None, suppress_stdout_stderror: bool = True, + add_encoders: Optional[dict] = None, **prophet_kwargs, ): """Facebook Prophet @@ -71,13 +72,33 @@ def __init__( Egypt (EG), China (CN), and Russia (RU). suppress_stdout_stderror Optionally suppress the log output produced by Prophet during training. + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. prophet_kwargs Some optional keyword arguments for Prophet. For information about the parameters see: `The Prophet source code `_. """ - super().__init__() + super().__init__(add_encoders=add_encoders) self._auto_seasonalities = self._extract_auto_seasonality(prophet_kwargs) diff --git a/darts/models/forecasting/regression_model.py b/darts/models/forecasting/regression_model.py index e76d159679..d50b0cb0b3 100644 --- a/darts/models/forecasting/regression_model.py +++ b/darts/models/forecasting/regression_model.py @@ -27,7 +27,6 @@ if their static covariates do not have the same size, the shorter ones are padded with 0 valued features. """ -import math from collections import OrderedDict from typing import List, Optional, Sequence, Tuple, Union @@ -72,13 +71,13 @@ def __init__( ---------- lags Lagged target values used to predict the next time step. If an integer is given the last `lags` past lags - are used (from -1 backward). Otherwise a list of integers with lags is required (each lag must be < 0). + are used (from -1 backward). Otherwise, a list of integers with lags is required (each lag must be < 0). lags_past_covariates Number of lagged past_covariates values used to predict the next time step. If an integer is given the last `lags_past_covariates` past lags are used (inclusive, starting from lag -1). Otherwise a list of integers with lags < 0 is required. lags_future_covariates - Number of lagged future_covariates values used to predict the next time step. If an tuple (past, future) is + Number of lagged future_covariates values used to predict the next time step. If a tuple (past, future) is given the last `past` lags in the past are used (inclusive, starting from lag -1) along with the first `future` future lags (starting from 0 - the prediction time - up to `future - 1` included). Otherwise a list of integers with lags is required. @@ -243,31 +242,31 @@ def __init__( self.pred_dim = self.output_chunk_length if self.multi_models else 1 @property - def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: - lags_covariates = { - lag for key in ["past", "future"] for lag in self.lags.get(key, []) - } - if lags_covariates: - # for lags < 0 we need to take `n` steps backwards from past and/or historic future covariates - # for minimum lag = -1 -> steps_back_inclusive = 1 - # inclusive means n steps back including the end of the target series - n_steps_back_inclusive = abs(min(min(lags_covariates), 0)) - # for lags >= 0 we need to take `n` steps ahead from future covariates - # for maximum lag = 0 -> output_chunk_length = 1 - # exclusive means n steps ahead after the last step of the target series - n_steps_ahead_exclusive = max(max(lags_covariates), 0) + 1 - takes_past_covariates = "past" in self.lags - takes_future_covariates = "future" in self.lags - else: - n_steps_back_inclusive = 0 - n_steps_ahead_exclusive = 0 - takes_past_covariates = False - takes_future_covariates = False + def _model_encoder_settings( + self, + ) -> Tuple[int, int, bool, bool, Optional[List[int]], Optional[List[int]]]: + target_lags = self.lags.get("target", [0]) + lags_past_covariates = self.lags.get("past", None) + if lags_past_covariates is not None: + lags_past_covariates = [ + min(lags_past_covariates) + - int(not self.multi_models) * (self.output_chunk_length - 1), + max(lags_past_covariates), + ] + lags_future_covariates = self.lags.get("future", None) + if lags_future_covariates is not None: + lags_future_covariates = [ + min(lags_future_covariates) + - int(not self.multi_models) * (self.output_chunk_length - 1), + max(lags_future_covariates), + ] return ( - n_steps_back_inclusive, - n_steps_ahead_exclusive, - takes_past_covariates, - takes_future_covariates, + abs(min(target_lags)), + self.output_chunk_length, + lags_past_covariates is not None, + lags_future_covariates is not None, + lags_past_covariates, + lags_future_covariates, ) @property @@ -290,18 +289,6 @@ def extreme_lags(self): max_future_cov_lag, ) - def _get_encoders_n(self, n) -> int: - """Returns the `n` encoder prediction steps specific to RegressionModels. - This will generate slightly more past covariates than the minimum requirement when using past and future - covariate lags simultaneously. This is because encoders were written for TorchForecastingModels where we only - needed `n` future covariates. For RegressionModel we need `n + max_future_lag` - """ - _, n_steps_ahead, _, takes_future_covariates = self._model_encoder_settings - if not takes_future_covariates: - return n - else: - return n + (n_steps_ahead - 1) - @property def min_train_series_length(self) -> int: return max( @@ -576,7 +563,6 @@ def predict( ) # prediction preprocessing - covariates = { "past": (past_covariates, self.lags.get("past")), "future": (future_covariates, self.lags.get("future")), @@ -594,54 +580,45 @@ def predict( covariate_matrices = {} # dictionary containing covariate lags relative to minimum covariate lag relative_cov_lags = {} - # number of prediction steps given forecast horizon and output_chunk_length - n_pred_steps = math.ceil(n / self.output_chunk_length) for cov_type, (covs, lags) in covariates.items(): - if covs is not None: - relative_cov_lags[cov_type] = np.array(lags) - lags[0] - covariate_matrices[cov_type] = [] - for idx, (ts, cov) in enumerate(zip(series, covs)): - # calculating first and last prediction time steps - first_pred_ts = ts.end_time() + 1 * ts.freq - last_pred_ts = ( - ( - first_pred_ts - + ((n_pred_steps - 1) * self.output_chunk_length) * ts.freq - ) - if self.multi_models - else (first_pred_ts + (n - 1) * ts.freq) + if covs is None: + continue + + relative_cov_lags[cov_type] = np.array(lags) - lags[0] + covariate_matrices[cov_type] = [] + for idx, (ts, cov) in enumerate(zip(series, covs)): + # how many steps to go back from end of target series for start of covariates + steps_back = -(min(lags) + 1) + shift + lags_diff = max(lags) - min(lags) + 1 + # over how many steps the covariates range + n_steps = lags_diff + max(0, n - self.output_chunk_length) + shift + + # calculate first and last required covariate time steps + start_ts = ts.end_time() - ts.freq * steps_back + end_ts = start_ts + ts.freq * (n_steps - 1) + + # check for sufficient covariate data + if not (cov.start_time() <= start_ts and cov.end_time() >= end_ts): + raise_log( + ValueError( + f"The corresponding {cov_type}_covariate of the series at index {idx} isn't sufficiently " + f"long. Given horizon `n={n}`, `min(lags_{cov_type}_covariates)={lags[0]}`, " + f"`max(lags_{cov_type}_covariates)={lags[-1]}` and " + f"`output_chunk_length={self.output_chunk_length}`, the {cov_type}_covariate has to range " + f"from {start_ts} until {end_ts} (inclusive), but it ranges only from {cov.start_time()} " + f"until {cov.end_time()}." + ), + logger=logger, ) - # calculating first and last required time steps - first_req_ts = ( - first_pred_ts + (lags[0] - shift) * ts.freq - ) # shift lags if using one_shot - last_req_ts = last_pred_ts + (lags[-1] - shift) * ts.freq - # check for sufficient covariate data - raise_if_not( - cov.start_time() <= first_req_ts - and cov.end_time() >= last_req_ts, - f"The corresponding {cov_type}_covariate of the series at index {idx} isn't sufficiently long. " - f"Given horizon `n={n}`, `min(lags_{cov_type}_covariates)={lags[0]}`, " - f"`max(lags_{cov_type}_covariates)={lags[-1]}` and " - f"`output_chunk_length={self.output_chunk_length}`\n" - f"the {cov_type}_covariate has to range from {first_req_ts} until {last_req_ts} (inclusive), " - f"but it ranges only from {cov.start_time()} until {cov.end_time()}.", - ) - - # Note: we use slice() rather than the [] operator because - # for integer-indexed series [] does not act on the time index. - last_req_ts = ( - # For range indexes, we need to make the end timestamp inclusive here - last_req_ts + ts.freq - if ts.has_range_index - else last_req_ts - ) - covariate_matrices[cov_type].append( - cov.slice(first_req_ts, last_req_ts).values(copy=False) - ) + # use slice() instead of [] as for integer-indexed series [] does not act on time index + # for range indexes, we make the end timestamp inclusive here + end_ts = end_ts + ts.freq if ts.has_range_index else end_ts + covariate_matrices[cov_type].append( + cov.slice(start_ts, end_ts).values(copy=False) + ) - covariate_matrices[cov_type] = np.stack(covariate_matrices[cov_type]) + covariate_matrices[cov_type] = np.stack(covariate_matrices[cov_type]) series_matrix = None if "target" in self.lags: @@ -661,20 +638,30 @@ def predict( covariate_matrices[cov_type] = np.repeat(data, num_samples, axis=0) # prediction predictions = [] + last_step_shift = 0 + # t_pred indicates the number of time steps after the first prediction for t_pred in range(0, n, step): + # in case of autoregressive forecast `(t_pred > 0)` and if `n` is not a round multiple of `step`, + # we have to step back `step` from `n` in the last iteration + if 0 < n - t_pred < step and t_pred > 0: + last_step_shift = t_pred - (n - step) + t_pred = n - step + np_X = [] # retrieve target lags if "target" in self.lags: - - target_matrix = ( - np.concatenate([series_matrix, *predictions], axis=1) - if predictions - else series_matrix - ) + if predictions: + series_matrix = np.concatenate( + [series_matrix, predictions[-1]], axis=1 + ) np_X.append( - target_matrix[ - :, [lag - shift for lag in self.lags["target"]] + series_matrix[ + :, + [ + lag - (shift + last_step_shift) + for lag in self.lags["target"] + ], ].reshape(len(series) * num_samples, -1) ) # retrieve covariate lags, enforce order (dict only preserves insertion order for python 3.6+) @@ -694,7 +681,7 @@ def predict( prediction = self._predict_and_sample(X, num_samples, **kwargs) # prediction shape (n_series * n_samples, output_chunk_length, n_components) # append prediction to final predictions - predictions.append(prediction) + predictions.append(prediction[:, last_step_shift:]) # concatenate and use first n points as prediction predictions = np.concatenate(predictions, axis=1)[:, :n] diff --git a/darts/models/forecasting/sf_auto_arima.py b/darts/models/forecasting/sf_auto_arima.py index 38e63244f7..66e53876a2 100644 --- a/darts/models/forecasting/sf_auto_arima.py +++ b/darts/models/forecasting/sf_auto_arima.py @@ -15,7 +15,9 @@ class StatsForecastAutoARIMA(FutureCovariatesLocalForecastingModel): - def __init__(self, *autoarima_args, **autoarima_kwargs): + def __init__( + self, *autoarima_args, add_encoders: Optional[dict] = None, **autoarima_kwargs + ): """Auto-ARIMA based on `Statsforecasts package `_. @@ -35,6 +37,26 @@ def __init__(self, *autoarima_args, **autoarima_kwargs): Positional arguments for ``statsforecasts.models.AutoARIMA``. autoarima_kwargs Keyword arguments for ``statsforecasts.models.AutoARIMA``. + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. Examples -------- @@ -45,7 +67,7 @@ def __init__(self, *autoarima_args, **autoarima_kwargs): >>> model.fit(series[:-36]) >>> pred = model.predict(36, num_samples=100) """ - super().__init__() + super().__init__(add_encoders=add_encoders) self.model = SFAutoARIMA(*autoarima_args, **autoarima_kwargs) def __str__(self): diff --git a/darts/models/forecasting/sf_ets.py b/darts/models/forecasting/sf_ets.py index a62f20766d..2f5f622d90 100644 --- a/darts/models/forecasting/sf_ets.py +++ b/darts/models/forecasting/sf_ets.py @@ -14,7 +14,7 @@ class StatsForecastETS(FutureCovariatesLocalForecastingModel): - def __init__(self, *ets_args, **ets_kwargs): + def __init__(self, *ets_args, add_encoders: Optional[dict] = None, **ets_kwargs): """ETS based on `Statsforecasts package `_. @@ -40,6 +40,26 @@ def __init__(self, *ets_args, **ets_kwargs): For instance, "ANN" means additive error, no trend and no seasonality. Furthermore, the character "Z" is a placeholder telling statsforecast to search for the best model using AICs. Default: "ZZZ". + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. Examples -------- @@ -50,7 +70,7 @@ def __init__(self, *ets_args, **ets_kwargs): >>> model.fit(series[:-36]) >>> pred = model.predict(36) """ - super().__init__() + super().__init__(add_encoders=add_encoders) self.model = ETS(*ets_args, **ets_kwargs) def __str__(self): diff --git a/darts/models/forecasting/torch_forecasting_model.py b/darts/models/forecasting/torch_forecasting_model.py index 52713b8fb5..a7a26dbc8f 100644 --- a/darts/models/forecasting/torch_forecasting_model.py +++ b/darts/models/forecasting/torch_forecasting_model.py @@ -1662,7 +1662,9 @@ def _verify_past_future_covariates(self, past_covariates, future_covariates): ) @property - def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: + def _model_encoder_settings( + self, + ) -> Tuple[int, int, bool, bool, Optional[List[int]], Optional[List[int]]]: input_chunk_length = self.input_chunk_length output_chunk_length = self.output_chunk_length takes_past_covariates = True @@ -1672,6 +1674,8 @@ def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: output_chunk_length, takes_past_covariates, takes_future_covariates, + None, + None, ) @property @@ -1747,7 +1751,9 @@ def _verify_past_future_covariates(self, past_covariates, future_covariates): ) @property - def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: + def _model_encoder_settings( + self, + ) -> Tuple[int, int, bool, bool, Optional[List[int]], Optional[List[int]]]: input_chunk_length = self.input_chunk_length output_chunk_length = self.output_chunk_length takes_past_covariates = False @@ -1757,6 +1763,8 @@ def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: output_chunk_length, takes_past_covariates, takes_future_covariates, + None, + None, ) @property @@ -1824,7 +1832,9 @@ def _verify_past_future_covariates(self, past_covariates, future_covariates): ) @property - def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: + def _model_encoder_settings( + self, + ) -> Tuple[int, int, bool, bool, Optional[List[int]], Optional[List[int]]]: input_chunk_length = self.input_chunk_length output_chunk_length = self.output_chunk_length takes_past_covariates = False @@ -1834,6 +1844,8 @@ def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: output_chunk_length, takes_past_covariates, takes_future_covariates, + None, + None, ) @property @@ -1898,7 +1910,9 @@ def _verify_past_future_covariates(self, past_covariates, future_covariates): pass @property - def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: + def _model_encoder_settings( + self, + ) -> Tuple[int, int, bool, bool, Optional[List[int]], Optional[List[int]]]: input_chunk_length = self.input_chunk_length output_chunk_length = self.output_chunk_length takes_past_covariates = True @@ -1908,6 +1922,8 @@ def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: output_chunk_length, takes_past_covariates, takes_future_covariates, + None, + None, ) @property @@ -1973,7 +1989,9 @@ def _verify_predict_sample(self, predict_sample: Tuple): raise NotImplementedError() @property - def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: + def _model_encoder_settings( + self, + ) -> Tuple[int, int, bool, bool, Optional[List[int]], Optional[List[int]]]: input_chunk_length = self.input_chunk_length output_chunk_length = self.output_chunk_length takes_past_covariates = True @@ -1983,6 +2001,8 @@ def _model_encoder_settings(self) -> Tuple[int, int, bool, bool]: output_chunk_length, takes_past_covariates, takes_future_covariates, + None, + None, ) @property diff --git a/darts/models/forecasting/varima.py b/darts/models/forecasting/varima.py index c9462fd062..83fc318330 100644 --- a/darts/models/forecasting/varima.py +++ b/darts/models/forecasting/varima.py @@ -25,7 +25,14 @@ class VARIMA(TransferableFutureCovariatesLocalForecastingModel): - def __init__(self, p: int = 1, d: int = 0, q: int = 0, trend: Optional[str] = None): + def __init__( + self, + p: int = 1, + d: int = 0, + q: int = 0, + trend: Optional[str] = None, + add_encoders: Optional[dict] = None, + ): """VARIMA Parameters @@ -44,8 +51,28 @@ def __init__(self, p: int = 1, d: int = 0, q: int = 0, trend: Optional[str] = No Parameter controlling the deterministic trend. 'n' indicates no trend, 'c' a constant term, 't' linear trend in time, and 'ct' includes both. Default is 'c' for models without integration, and no trend for models with integration. + add_encoders + A large number of future covariates can be automatically generated with `add_encoders`. + This can be done by adding multiple pre-defined index encoders and/or custom user-made functions that + will be used as index encoders. Additionally, a transformer such as Darts' :class:`Scaler` can be added to + transform the generated covariates. This happens all under one hood and only needs to be specified at + model creation. + Read :meth:`SequentialEncoder ` to find out more about + ``add_encoders``. Default: ``None``. An example showing some of ``add_encoders`` features: + + .. highlight:: python + .. code-block:: python + + add_encoders={ + 'cyclic': {'future': ['month']}, + 'datetime_attribute': {'future': ['hour', 'dayofweek']}, + 'position': {'future': ['relative']}, + 'custom': {'future': [lambda idx: (idx.year - 1950) / 50]}, + 'transformer': Scaler() + } + .. """ - super().__init__() + super().__init__(add_encoders=add_encoders) self.p = p self.d = d self.q = q diff --git a/darts/tests/dataprocessing/encoders/test_covariate_index_generators.py b/darts/tests/dataprocessing/encoders/test_covariate_index_generators.py index 544db21957..19fb647282 100644 --- a/darts/tests/dataprocessing/encoders/test_covariate_index_generators.py +++ b/darts/tests/dataprocessing/encoders/test_covariate_index_generators.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +import pytest from darts import TimeSeries from darts.dataprocessing.encoders.encoder_base import ( @@ -107,18 +108,34 @@ def helper_test_index_generator_train(self, ig: CovariatesIndexGenerator): self.assertTrue(idx.equals(self.cov_time_train_short.time_index)) # generated index must be equal to input target index when no covariates are defined idx, _ = ig.generate_train_idx(self.target_time, None) - self.assertTrue(idx.equals(self.cov_time_train.time_index)) + self.assertEqual(idx[0], self.target_time.start_time()) + if isinstance(ig, PastCovariatesIndexGenerator): + self.assertEqual( + idx[-1], + self.target_time.end_time() + - self.output_chunk_length * self.target_time.freq, + ) + else: + self.assertEqual(idx[-1], self.target_time.end_time()) # integer index # generated index must be equal to input covariate index idx, _ = ig.generate_train_idx(self.target_int, self.cov_int_train) self.assertTrue(idx.equals(self.cov_int_train.time_index)) # generated index must be equal to input covariate index - idx, _ = ig.generate_train_idx(self.target_time, self.cov_int_train_short) + idx, _ = ig.generate_train_idx(self.target_int, self.cov_int_train_short) self.assertTrue(idx.equals(self.cov_int_train_short.time_index)) # generated index must be equal to input target index when no covariates are defined idx, _ = ig.generate_train_idx(self.target_int, None) - self.assertTrue(idx.equals(self.cov_int_train.time_index)) + self.assertEqual(idx[0], self.target_int.start_time()) + if isinstance(ig, PastCovariatesIndexGenerator): + self.assertEqual( + idx[-1], + self.target_int.end_time() + - self.output_chunk_length * self.target_int.freq, + ) + else: + self.assertEqual(idx[-1], self.target_int.end_time()) def helper_test_index_generator_inference(self, ig, is_past=False): """ @@ -177,6 +194,130 @@ def helper_test_index_generator_inference(self, ig, is_past=False): ) self.assertTrue(idx.equals(self.cov_int_inf_long.time_index)) + def helper_test_index_generator_creation(self, ig_cls, is_past=False): + # invalid parameter sets + with pytest.raises(ValueError): + _ = ig_cls( + output_chunk_length=3, + ) + + with pytest.raises(ValueError): + _ = ig_cls( + output_chunk_length=3, + lags_covariates=[-1], + ) + + # valid parameter sets + # TorchForecastingModel scenario + _ = ig_cls( + input_chunk_length=3, + output_chunk_length=3, + ) + # RegressionModel scenario + _ = ig_cls( + input_chunk_length=3, + output_chunk_length=3, + lags_covariates=[-1], + ) + # LocalForecastingModel scenario, or model agnostic (only supported by FutureCovariatesIndexGenerator) + if not is_past: + _ = ig_cls() + + def test_past_index_generator_creation(self): + # test parameter scenarios + self.helper_test_index_generator_creation( + ig_cls=PastCovariatesIndexGenerator, is_past=True + ) + + # ==> test failures + # one lag is >= 0 (not possible for past covariates) + with pytest.raises(ValueError): + _ = PastCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[-1, 1], + ) + with pytest.raises(ValueError): + _ = PastCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[0, -1], + ) + + min_lag, max_lag = -2, -1 + ig = PastCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[min_lag, max_lag], + ) + self.assertEqual(ig.shift_start, min_lag + 1) + self.assertEqual(ig.shift_end, max_lag + 1) + + min_lag, max_lag = -1, -1 + ig = PastCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[min_lag, max_lag], + ) + self.assertEqual(ig.shift_start, min_lag + 1) + self.assertEqual(ig.shift_end, max_lag + 1) + + # check that min/max lags are extracted from list of lags + min_lag, max_lag = -10, -3 + ig = PastCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[-5, min_lag, max_lag, -4], + ) + self.assertEqual(ig.shift_start, min_lag + 1) + self.assertEqual(ig.shift_end, max_lag + 1) + + def test_future_index_generator_creation(self): + # test parameter scenarios + self.helper_test_index_generator_creation( + ig_cls=FutureCovariatesIndexGenerator, is_past=False + ) + + # future covariates index generator (ig) can technically be used like a past covariates ig + min_lag, max_lag = -2, -1 + ig = FutureCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[min_lag, max_lag], + ) + self.assertEqual(ig.shift_start, min_lag + 1) + self.assertEqual(ig.shift_end, max_lag + 1) + + min_lag, max_lag = -1, -1 + ig = FutureCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[min_lag, max_lag], + ) + self.assertEqual(ig.shift_start, min_lag + 1) + self.assertEqual(ig.shift_end, max_lag + 1) + + # different to past covariates ig, future ig can take positive and negative lags + min_lag, max_lag = -2, 1 + ig = FutureCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[min_lag, max_lag], + ) + self.assertEqual(ig.shift_start, min_lag + 1) + # when `max_lag` >= 0, we add one step to `shift_end`, as future lags start at 0 meaning first prediction step + self.assertEqual(ig.shift_end, max_lag + 1) + + # check that min/max lags are extracted from list of lags + min_lag, max_lag = -10, 5 + ig = FutureCovariatesIndexGenerator( + 1, + 1, + lags_covariates=[-5, min_lag, max_lag, -1], + ) + self.assertEqual(ig.shift_start, min_lag + 1) + self.assertEqual(ig.shift_end, max_lag + 1) + def test_past_index_generator(self): ig = PastCovariatesIndexGenerator( self.input_chunk_length, self.output_chunk_length @@ -185,6 +326,105 @@ def test_past_index_generator(self): self.helper_test_index_generator_train(ig) self.helper_test_index_generator_inference(ig, is_past=True) + def test_past_index_generator_with_lags(self): + icl = self.input_chunk_length + ocl = self.output_chunk_length + freq = self.target_time.freq + target = self.target_time + + def test_routine_train( + self, icl, ocl, min_lag, max_lag, start_expected, end_expected + ): + idxg = PastCovariatesIndexGenerator( + icl, + ocl, + lags_covariates=[min_lag, max_lag], + ) + idx, _ = idxg.generate_train_idx(target, None) + self.assertEqual(idx[0], pd.Timestamp(start_expected, freq=freq)) + self.assertEqual(idx[-1], pd.Timestamp(end_expected, freq=freq)) + # check case 0: we give covariates, index will always be the covariate time index + idx, _ = idxg.generate_train_idx(target, self.cov_time_train) + self.assertTrue(idx.equals(self.cov_time_train.time_index)) + return idxg + + def test_routine_inf(self, idxg, n, start_expected, end_expected): + idx, _ = idxg.generate_inference_idx(n, target, None) + self.assertEqual(idx[0], pd.Timestamp(start_expected, freq=freq)) + self.assertEqual(idx[-1], pd.Timestamp(end_expected, freq=freq)) + # check case 0: we give covariates, index will always be the covariate time index + idx, _ = idxg.generate_inference_idx(n, target, self.cov_time_inf_short) + self.assertTrue(idx.equals(self.cov_time_inf_short.time_index)) + + # lags are required for RegressionModels + # case 1: abs(min_lags) == icl and abs(max_lag) == -1: + # will give identical results as without setting lags + min_lag = -12 # = -icl + max_lag = -1 + expected_start = "2000-01-01" + expected_end = "2001-06-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + self.helper_test_index_types(ig) + self.helper_test_index_generator_train(ig) + self.helper_test_index_generator_inference(ig, is_past=True) + # check inference for n <= ocl + expected_start = "2001-01-01" + expected_end = "2001-12-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-01-01") + + # case 2: abs(min_lag) < icl and abs(max_lag) == -1: + # the start time of covariates begins before target start + min_lag, max_lag = -11, -1 + expected_start = "2000-02-01" + expected_end = "2001-06-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + # check inference for n <= ocl + expected_start = "2001-02-01" + expected_end = "2001-12-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-01-01") + + # case 3: abs(min_lag) > icl and abs(max_lag) == -1: + # the start time of covariates begins before target start + min_lag, max_lag = -13, -1 + expected_start = "1999-12-01" + expected_end = "2001-06-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + # check inference for n <= ocl + expected_start = "2000-12-01" + expected_end = "2001-12-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-01-01") + + # case 4: abs(min_lag) > icl and abs(max_lag) > -1: + # the start time of covariates begins before target start + min_lag, max_lag = -13, -2 + expected_start = "1999-12-01" + expected_end = "2001-05-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + # check inference for n <= ocl + expected_start = "2000-12-01" + expected_end = "2001-11-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2001-12-01") + def test_future_index_generator(self): ig = FutureCovariatesIndexGenerator( self.input_chunk_length, self.output_chunk_length @@ -192,3 +432,200 @@ def test_future_index_generator(self): self.helper_test_index_types(ig) self.helper_test_index_generator_train(ig) self.helper_test_index_generator_inference(ig, is_past=False) + + def test_future_index_generator_with_lags(self): + icl = self.input_chunk_length + ocl = self.output_chunk_length + freq = self.target_time.freq + target = self.target_time + + def test_routine_train( + self, icl, ocl, min_lag, max_lag, start_expected, end_expected + ): + idxg = FutureCovariatesIndexGenerator( + icl, + ocl, + lags_covariates=[min_lag, max_lag], + ) + idx, _ = idxg.generate_train_idx(target, None) + self.assertEqual(idx[0], pd.Timestamp(start_expected, freq=freq)) + self.assertEqual(idx[-1], pd.Timestamp(end_expected, freq=freq)) + # check case 0: we give covariates, index will always be the covariate time index + idx, _ = idxg.generate_train_idx(target, self.cov_time_train) + self.assertTrue(idx.equals(self.cov_time_train.time_index)) + return idxg + + def test_routine_inf(self, idxg, n, start_expected, end_expected): + idx, _ = idxg.generate_inference_idx(n, target, None) + self.assertTrue(idx[0], pd.Timestamp(start_expected, freq=freq)) + self.assertTrue(idx[-1], pd.Timestamp(end_expected, freq=freq)) + # check case 0: we give covariates, index will always be the covariate time index + idx, _ = idxg.generate_inference_idx(n, target, self.cov_time_inf_short) + self.assertTrue(idx.equals(self.cov_time_inf_short.time_index)) + + # INFO: test cases 1, 2, and 3 only have lags in the past which yields identical results as using a + # PastCovariatesIndexGenerator + # case 1: abs(min_lag) < icl and abs(max_lag) == -1: + # the start time of covariates begins before target start + min_lag, max_lag = -11, -1 + expected_start = "2000-02-01" + expected_end = "2001-06-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + # check inference for n <= ocl + expected_start = "2001-02-01" + expected_end = "2001-12-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-01-01") + + # case 2: abs(min_lag) > icl and abs(max_lag) == -1: + # the start time of covariates begins before target start + min_lag, max_lag = -13, -1 + expected_start = "1999-12-01" + expected_end = "2001-06-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + # check inference for n <= ocl + expected_start = "2000-12-01" + expected_end = "2001-12-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-01-01") + + # case 3: abs(min_lag) > icl and abs(max_lag) > -1: + # the start time of covariates begins before target start + min_lag, max_lag = -13, -2 + expected_start = "1999-12-01" + expected_end = "2001-05-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + # check inference for n <= ocl + expected_start = "2000-12-01" + expected_end = "2001-11-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2001-12-01") + + # INFO: the following test cases have lags in the future which is different to PastCovariatesIndexGenerator + # case 4: abs(min_lags) == icl and max_lag == (ocl - 1): + # will give identical results as without setting lags + min_lag = -12 # -icl + max_lag = 5 # (ocl - 1) + expected_start = "2000-01-01" + expected_end = "2001-12-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + self.helper_test_index_types(ig) + self.helper_test_index_generator_train(ig) + self.helper_test_index_generator_inference(ig, is_past=False) + # check inference for n <= ocl + expected_start = "2001-01-01" + expected_end = "2002-06-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-07-01") + + # case 5: abs(min_lag) == icl and max_lag < (ocl - 1) + # the end of covariates should be one time step after beginning of last output chunk with max_lag = 0 + min_lag, max_lag = -12, 0 + expected_start = "2000-01-01" + expected_end = "2001-07-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + expected_start = "2001-01-01" + expected_end = "2002-01-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-02-01") + + # case 6: abs(min_lag) == icl and max_lag > (ocl - 1) + # the end of covariates is after the end of target series with max_lag = (ocl - 1) + 1 + min_lag, max_lag = -12, 17 + expected_start = "2000-01-01" + expected_end = "2002-12-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + expected_start = "2001-01-01" + expected_end = "2003-01-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2003-02-01") + + # case 7: min_lag >= 0 and max_lag <= (ocl - 1) + # only future part of future covariates (no historical part) + min_lag, max_lag = 0, 2 + expected_start = "2001-01-01" + expected_end = "2001-09-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + expected_start = "2002-01-01" + expected_end = "2002-03-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-04-01") + + # case 8: min_lag >= 0 and max_lag > (ocl - 1) + # only future part of future covariates (no historical part) + min_lag, max_lag = 0, 17 + expected_start = "2001-01-01" + expected_end = "2002-12-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + expected_start = "2002-01-01" + expected_end = "2003-01-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2003-02-01") + + # case 9: abs(min_lag) > icl and max_lag > (ocl - 1) + min_lag, max_lag = -13, 17 + expected_start = "1999-12-01" + expected_end = "2002-12-01" + ig = test_routine_train( + self, icl, ocl, min_lag, max_lag, expected_start, expected_end + ) + expected_start = "2000-12-01" + expected_end = "2002-03-01" + test_routine_inf(self, ig, 1, expected_start, expected_end) + test_routine_inf(self, ig, ocl, expected_start, expected_end) + # check inference for n > ocl + test_routine_inf(self, ig, ocl + 1, expected_start, "2002-04-01") + + def test_future_index_generator_local(self): + # test model agnostic scenario (also for LocalForecastingModels) + freq = self.target_time.freq + target = self.target_time + + idxg = FutureCovariatesIndexGenerator() + idx, _ = idxg.generate_train_idx(target=target, covariates=None) + self.assertTrue(idx.equals(target.time_index)) + idx, _ = idxg.generate_train_idx(target=target, covariates=self.cov_time_train) + self.assertTrue(idx.equals(self.cov_time_train.time_index)) + + n = 10 + idx, _ = idxg.generate_inference_idx(n=n, target=target, covariates=None) + self.assertEqual(idx.freq, freq) + self.assertEqual(idx[0], target.end_time() + 1 * freq) + self.assertEqual(idx[-1], target.end_time() + n * freq) + + idx, _ = idxg.generate_inference_idx( + n=n, target=target, covariates=self.cov_int_inf_short + ) + self.assertTrue(idx.equals(self.cov_int_inf_short.time_index)) diff --git a/darts/tests/dataprocessing/encoders/test_encoders.py b/darts/tests/dataprocessing/encoders/test_encoders.py index 4b1ec66300..85ae35b6fc 100644 --- a/darts/tests/dataprocessing/encoders/test_encoders.py +++ b/darts/tests/dataprocessing/encoders/test_encoders.py @@ -416,7 +416,11 @@ def test_routine(encoder, merge_covs: bool, comps_expected: pd.Index): covs_train2 = encoder.encode_train( target=ts, covariates=covs_train, merge_covariates=merge_covs ) - assert covs_train2 == covs_train + if merge_covs: + assert covs_train2 == covs_train + else: + overlap = covs_train.slice_intersect(covs_train2) + assert covs_train2 == overlap # we can use the output of `encode_train()` as input for `encode_inference()`. The encoded components # are dropped internally and appended again at the end @@ -429,11 +433,13 @@ def test_routine(encoder, merge_covs: bool, comps_expected: pd.Index): # We get only the minimum required time spans with `merge_covariates=False` as input covariates will # are not in output of `encode_train()/inference()` else: + overlap = covs_inf.slice_intersect(covs_inf2) if isinstance( encoder.index_generator, PastCovariatesIndexGenerator ): assert len(covs_inf2) == input_chunk_length assert covs_inf2.end_time() == ts.end_time() + assert covs_inf2 == overlap else: assert ( len(covs_inf2) == input_chunk_length + output_chunk_length @@ -442,6 +448,8 @@ def test_routine(encoder, merge_covs: bool, comps_expected: pd.Index): covs_inf2.end_time() == ts.end_time() + ts.freq * output_chunk_length ) + overlap_inf = covs_inf2.slice_intersect(overlap) + assert overlap_inf == overlap # we can use the output of `encode_inference()` as input for `encode_inference()` and get the # same results (encoded components get overwritten) @@ -728,9 +736,8 @@ def test_integer_positional_encoder(self): ts = tg.linear_timeseries(length=24, freq="MS") input_chunk_length = 12 output_chunk_length = 6 - # ===> test relative position encoder <=== - encoder_params = {"position": {"past": ["relative"]}} + encoder_params = {"position": {"past": ["relative"], "future": ["relative"]}} encs = SequentialEncoder( add_encoders=encoder_params, input_chunk_length=input_chunk_length, @@ -738,47 +745,113 @@ def test_integer_positional_encoder(self): takes_past_covariates=True, takes_future_covariates=True, ) + # relative encoder takes the end of the training series as reference + vals = np.arange(-len(ts) + 1, 1).reshape((len(ts), 1)) + + pc1, fc1 = encs.encode_train(ts) + self.assertTrue( + pc1.time_index.equals(ts.time_index[:-output_chunk_length]) + and (pc1.values() == vals[:-output_chunk_length]).all() + ) + self.assertTrue( + fc1.time_index.equals(ts.time_index) and (fc1.values() == vals).all() + ) - t1, _ = encs.encode_train(ts) - t2, _ = encs.encode_train( + pc2, fc2 = encs.encode_train( TimeSeries.from_times_and_values( ts.time_index[:20] + ts.freq, ts[:20].values() ) ) - t3, _ = encs.encode_train( + self.assertTrue( + (pc2.time_index.equals(ts.time_index[: 20 - output_chunk_length] + ts.freq)) + and (pc2.values() == vals[-20:-output_chunk_length]).all() + ) + self.assertTrue( + fc2.time_index.equals(ts.time_index[:20] + ts.freq) + and (fc2.values() == vals[-20:]).all() + ) + + pc3, fc3 = encs.encode_train( TimeSeries.from_times_and_values( ts.time_index[:18] - ts.freq, ts[:18].values() ) ) - # relative encoder takes the end of the training series as reference - vals = np.arange(-len(ts) + 1, 1).reshape((len(ts), 1)) self.assertTrue( - (t1.time_index == ts.time_index).all() and (t1.values() == vals).all() + pc3.time_index.equals(ts.time_index[: 18 - output_chunk_length] - ts.freq) + and (pc3.values() == vals[-18:-output_chunk_length]).all() ) self.assertTrue( - (t2.time_index == ts.time_index[:20] + ts.freq).all() - and (t2.values() == vals[-20:]).all() - ) - self.assertTrue( - (t3.time_index == ts.time_index[:18] - ts.freq).all() - and (t3.values() == vals[-18:]).all() + fc3.time_index.equals(ts.time_index[:18] - ts.freq) + and (fc3.values() == vals[-18:]).all() ) + # quickly test inference encoding # n > output_chunk_length - t4, _ = encs.encode_inference(output_chunk_length + 1, ts) + n = output_chunk_length + 1 + pc4, fc4 = encs.encode_inference(n, ts) self.assertTrue( - (t4.values()[:, 0] == np.arange(-input_chunk_length + 1, 1 + 1)).all() + (pc4.univariate_values() == np.arange(-input_chunk_length + 1, 1 + 1)).all() + ) + self.assertTrue( + (fc4.univariate_values() == np.arange(-input_chunk_length + 1, 1 + n)).all() ) # n <= output_chunk_length - t5, _ = encs.encode_inference( - output_chunk_length - 1, + n = output_chunk_length - 1 + t5, fc5 = encs.encode_inference( + n, TimeSeries.from_times_and_values( ts.time_index[:20] + ts.freq, ts[:20].values() ), ) - # t5, _ = encs.encode_inference(output_chunk_length - 1, ts) self.assertTrue( - (t5.values()[:, 0] == np.arange(-input_chunk_length + 1, 0 + 1)).all() + (t5.univariate_values() == np.arange(-input_chunk_length + 1, 0 + 1)).all() + ) + self.assertTrue( + ( + fc5.univariate_values() + == np.arange(-input_chunk_length + 1, output_chunk_length + 1) + ).all() + ) + + # quickly test with lags + min_pc_lag = -input_chunk_length - 2 # = -14 + max_pc_lag = -6 + min_fc_lag = 2 + max_fc_lag = 8 + encs = SequentialEncoder( + add_encoders=encoder_params, + input_chunk_length=input_chunk_length, + output_chunk_length=output_chunk_length, + takes_past_covariates=True, + takes_future_covariates=True, + lags_past_covariates=[min_pc_lag, max_pc_lag], + lags_future_covariates=[min_fc_lag, max_fc_lag], + ) + pc1, fc1 = encs.encode_train(ts) + self.assertTrue( + pc1.start_time() == pd.Timestamp("1999-11-01", freq=ts.freq) + and pc1.end_time() == pd.Timestamp("2001-01-01", freq=ts.freq) + and (pc1.univariate_values() == np.arange(-25, -10)).all() + and pc1[ts.start_time()].univariate_values()[0] == -23 + ) + self.assertTrue( + fc1.start_time() == pd.Timestamp("2001-03-01", freq=ts.freq) + and fc1.end_time() == pd.Timestamp("2002-03-01", freq=ts.freq) + and (fc1.univariate_values() == np.arange(-9, 4)).all() + and fc1[ts.end_time()].univariate_values()[0] == 0 + ) + + n = 2 + pc2, fc2 = encs.encode_inference(n=n, target=ts) + self.assertTrue( + pc2.start_time() == pd.Timestamp("2000-11-01", freq=ts.freq) + and pc2.end_time() == pd.Timestamp("2001-07-01", freq=ts.freq) + and (pc2.univariate_values() == np.arange(-13, -4)).all() + ) + self.assertTrue( + fc2.start_time() == pd.Timestamp("2002-03-01", freq=ts.freq) + and fc2.end_time() == pd.Timestamp("2002-09-01", freq=ts.freq) + and (fc2.univariate_values() == np.arange(3, 10)).all() ) def test_callable_encoder(self): @@ -789,7 +862,10 @@ def test_callable_encoder(self): # ===> test callable index encoder <=== encoder_params = { - "custom": {"past": [lambda index: index.year, lambda index: index.year - 1]} + "custom": { + "past": [lambda index: index.year, lambda index: index.year - 1], + "future": [lambda index: index.year], + } } encs = SequentialEncoder( add_encoders=encoder_params, @@ -799,9 +875,34 @@ def test_callable_encoder(self): takes_future_covariates=True, ) - t1, _ = encs.encode_train(ts) - self.assertTrue((ts.time_index.year.values == t1.values()[:, 0]).all()) - self.assertTrue((ts.time_index.year.values - 1 == t1.values()[:, 1]).all()) + # train set + pc, fc = encs.encode_train(ts) + # past covariates + np.testing.assert_array_equal( + ts[:-output_chunk_length].time_index.year.values, pc.values()[:, 0] + ) + np.testing.assert_array_equal( + ts[:-output_chunk_length].time_index.year.values - 1, pc.values()[:, 1] + ) + # future covariates + np.testing.assert_array_equal(ts.time_index.year.values, fc.values()[:, 0]) + + # inference set + pc, fc = encs.encode_inference(n=12, target=ts) + year_index = tg.generate_index( + start=ts.end_time() - ts.freq * (input_chunk_length - 1), + length=24, + freq=ts.freq, + ) + # past covariates + np.testing.assert_array_equal( + year_index[:-output_chunk_length].year.values, pc.values()[:, 0] + ) + np.testing.assert_array_equal( + year_index[:-output_chunk_length].year.values - 1, pc.values()[:, 1] + ) + # future covariates + np.testing.assert_array_equal(year_index.year.values, fc.values()[:, 0]) def test_transformer_single_series(self): def test_routine_cyclic(past_covs): @@ -912,10 +1013,11 @@ def test_transformer_multi_series(self): "transformer": Scaler(), } + ocl = 6 enc_base = SequentialEncoder( add_encoders=encoder_params, input_chunk_length=11, - output_chunk_length=6, + output_chunk_length=ocl, takes_past_covariates=True, takes_future_covariates=True, ) @@ -927,7 +1029,7 @@ def test_transformer_multi_series(self): # user supplied covariates should not be transformed self.assertTrue(fc[0]["cov"] == ts1) self.assertTrue(fc[1]["cov"] == ts2) - # check that first covariate series ranges from 0. to 1. and second from 0.5 to 1. + # check that first covariate series ranges from 0. to 1. and second from ~0.7 to 1. for covs, cov_name in zip( [pc, fc], ["darts_enc_pc_dta_minute", "darts_enc_fc_dta_minute"] ): @@ -937,15 +1039,23 @@ def test_transformer_multi_series(self): self.assertAlmostEqual( covs[0][cov_name].values(copy=False).max(), 1.0, delta=10e-9 ) - self.assertAlmostEqual( - covs[1][cov_name].values(copy=False).min(), 0.5, delta=10e-9 + self.assertEqual( + covs[0][cov_name].univariate_values(copy=False)[-4], + covs[1][cov_name].univariate_values(copy=False)[-4], ) + if "pc" in cov_name: + self.assertAlmostEqual( + covs[1][cov_name].values(copy=False).min(), 0.714, delta=1e-2 + ) + else: + self.assertAlmostEqual( + covs[1][cov_name].values(copy=False).min(), 0.5, delta=1e-2 + ) self.assertAlmostEqual( covs[1][cov_name].values(copy=False).max(), 1.0, delta=10e-9 ) - # check the same for inference (future covs will be identical to `encode_train()` as we give covariates - # as input) + # check the same for inference pc, fc = enc.encode_inference( n=6, target=[ts1, ts2], future_covariates=[ts1_inf, ts2_inf] ) @@ -953,24 +1063,26 @@ def test_transformer_multi_series(self): [pc, fc], ["darts_enc_pc_dta_minute", "darts_enc_fc_dta_minute"] ): for cov in covs: - self.assertAlmostEqual( - cov[cov_name].values(copy=False).min(), 0.5, delta=10e-9 - ) - self.assertAlmostEqual( - cov[cov_name].values(copy=False).max(), 1.0, delta=10e-9 - ) + if "pc" in cov_name: + self.assertEqual( + cov[cov_name][-(ocl + 1)].univariate_values()[0], 1.0 + ) + else: + self.assertEqual( + cov[cov_name][ts1.end_time()].univariate_values()[0], 1.0 + ) # check the same for only supplying single series as input pc, fc = enc.encode_inference(n=6, target=ts2, future_covariates=ts2_inf) for cov, cov_name in zip( [pc, fc], ["darts_enc_pc_dta_minute", "darts_enc_fc_dta_minute"] ): - self.assertAlmostEqual( - cov[cov_name].values(copy=False).min(), 0.5, delta=10e-9 - ) - self.assertAlmostEqual( - cov[cov_name].values(copy=False).max(), 1.0, delta=10e-9 - ) + if "pc" in cov_name: + self.assertEqual(cov[cov_name][-(ocl + 1)].univariate_values()[0], 1.0) + else: + self.assertEqual( + cov[cov_name][ts1.end_time()].univariate_values()[0], 1.0 + ) # ====> TEST Transformation starting from single-TimeSeries input: transformer is fit per component of a single # encoded series @@ -978,7 +1090,6 @@ def test_transformer_multi_series(self): pc, fc = enc.encode_train(ts2, future_covariates=ts2) # user supplied covariates should not be transformed self.assertTrue(fc["cov"] == ts2) - # check that first covariate series ranges from 0. to 1. and second from 0.5 to 1. for covs, cov_name in zip( [pc, fc], ["darts_enc_pc_dta_minute", "darts_enc_fc_dta_minute"] ): @@ -994,9 +1105,14 @@ def test_transformer_multi_series(self): for covs, cov_name in zip( [pc, fc], ["darts_enc_pc_dta_minute", "darts_enc_fc_dta_minute"] ): - self.assertAlmostEqual( - covs[0][cov_name].values(copy=False).min(), -1.0, delta=10e-9 - ) + if "pc" in cov_name: + self.assertAlmostEqual( + covs[0][cov_name].values(copy=False).min(), -2.5, delta=10e-9 + ) + else: + self.assertAlmostEqual( + covs[0][cov_name].values(copy=False).min(), -1.0, delta=10e-9 + ) self.assertAlmostEqual( covs[0][cov_name].values(copy=False).max(), 1.0, delta=10e-9 ) @@ -1007,8 +1123,7 @@ def test_transformer_multi_series(self): covs[1][cov_name].values(copy=False).max(), 1.0, delta=10e-9 ) - # check inference with single series (future covs will be identical to `encode_train()` as we give covariates - # as input) + # check inference with single series pc, fc = enc.encode_inference(n=6, target=ts2, future_covariates=ts2_inf) for cov, cov_name in zip( [pc, fc], ["darts_enc_pc_dta_minute", "darts_enc_fc_dta_minute"] @@ -1016,9 +1131,14 @@ def test_transformer_multi_series(self): self.assertAlmostEqual( cov[cov_name].values(copy=False).min(), 0.0, delta=10e-9 ) - self.assertAlmostEqual( - cov[cov_name].values(copy=False).max(), 1.0, delta=10e-9 - ) + if "pc" in cov_name: + self.assertAlmostEqual( + cov[cov_name].values(copy=False).max(), 2.5, delta=10e-9 + ) + else: + self.assertAlmostEqual( + cov[cov_name].values(copy=False).max(), 1.0, delta=10e-9 + ) # check the same for supplying multiple series as input pc, fc = enc.encode_inference( @@ -1031,9 +1151,14 @@ def test_transformer_multi_series(self): self.assertAlmostEqual( cov[cov_name].values(copy=False).min(), 0.0, delta=10e-9 ) - self.assertAlmostEqual( - cov[cov_name].values(copy=False).max(), 1.0, delta=10e-9 - ) + if "pc" in cov_name: + self.assertAlmostEqual( + cov[cov_name].values(copy=False).max(), 2.5, delta=10e-9 + ) + else: + self.assertAlmostEqual( + cov[cov_name].values(copy=False).max(), 1.0, delta=10e-9 + ) def helper_test_cyclic_encoder( self, @@ -1161,7 +1286,16 @@ def helper_test_encoder_single_train( encoded.append( encoder.encode_train(ts, cov, merge_covariates=merge_covariates) ) - self.assertTrue(encoded == result) + + expected_result = result + # when user does not give covariates, and a past covariate encoder is used, the generate train covariates are + # `output_chunk_length` steps shorter than the target series + if covariates[0] is None and isinstance( + encoder.index_generator, PastCovariatesIndexGenerator + ): + expected_result = [res[: -self.output_chunk_length] for res in result] + + self.assertTrue(encoded == expected_result) def helper_test_encoder_single_inference( self, diff --git a/darts/tests/models/forecasting/test_local_forecasting_models.py b/darts/tests/models/forecasting/test_local_forecasting_models.py index 3b7d4c2b18..63b5c9d82a 100644 --- a/darts/tests/models/forecasting/test_local_forecasting_models.py +++ b/darts/tests/models/forecasting/test_local_forecasting_models.py @@ -1,3 +1,4 @@ +import copy import os import shutil import tempfile @@ -6,6 +7,7 @@ import numpy as np import pandas as pd +import pytest from darts.datasets import AirPassengersDataset, IceCreamHeaterDataset from darts.logging import get_logger @@ -88,6 +90,15 @@ AutoARIMA(), ] +# test only a few models for encoder support reduce time +encoder_support_models = [ + VARIMA(1, 0, 0), + ARIMA(), + AutoARIMA(), + Prophet(), + KalmanForecaster(dim_x=30), +] + class LocalForecastingModelsTestCase(DartsBaseTestClass): @@ -266,6 +277,47 @@ def test_exogenous_variables_support(self): with self.assertRaises(ValueError): model.fit(target[1:], future_covariates=target[:-1]) + def test_encoders_support(self): + # test case with pd.DatetimeIndex + n = 3 + + target = self.ts_gaussian[:-3] + future_covariates = self.ts_gaussian + + add_encoders = {"custom": {"future": [lambda x: x.dayofweek]}} + + # test some models that do not support encoders + no_support_model_cls = [NaiveMean, Theta] + for model_cls in no_support_model_cls: + with pytest.raises(TypeError): + _ = model_cls(add_encoders=add_encoders) + + # test some models that support encoders + for model_object in encoder_support_models: + series = ( + target + if not isinstance(model_object, VARIMA) + else target.stack(target.map(np.log)) + ) + # test once with user supplied covariates, and once without + for fc in [future_covariates, None]: + model_params = { + k: vals + for k, vals in copy.deepcopy(model_object.model_params).items() + } + model_params["add_encoders"] = add_encoders + model = model_object.__class__(**model_params) + + # Test models with user supplied covariates + model.fit(series, future_covariates=fc) + + prediction = model.predict(n, future_covariates=fc) + self.assertTrue(len(prediction) == n) + + if isinstance(model, TransferableFutureCovariatesLocalForecastingModel): + prediction = model.predict(n, series=series, future_covariates=fc) + self.assertTrue(len(prediction) == n) + def test_dummy_series(self): values = np.random.uniform(low=-10, high=10, size=100) ts = TimeSeries.from_dataframe(pd.DataFrame({"V1": values})) diff --git a/darts/tests/models/forecasting/test_regression_models.py b/darts/tests/models/forecasting/test_regression_models.py index eed3e1ef42..02d9394d0f 100644 --- a/darts/tests/models/forecasting/test_regression_models.py +++ b/darts/tests/models/forecasting/test_regression_models.py @@ -686,11 +686,9 @@ def test_static_covs_addition(self): max_samples = 5 all_series_width = series1.n_components max_scovs_width = max( - [ - s.static_covariates_values(copy=False).reshape(1, -1).shape[1] - for s in all_series - if s.has_static_covariates - ] + s.static_covariates_values(copy=False).reshape(1, -1).shape[1] + for s in all_series + if s.has_static_covariates ) # no static covs @@ -1292,21 +1290,19 @@ def test_not_enough_covariates(self): max_samples_per_ts=1, ) + n = 10 # output_chunk_length, required past_offset, required future_offset - test_cases_multi_models = [ - (1, 0, 13), - (5, -4, 9), - (7, -2, 11), - ] - - test_cases_one_shot = [ + test_cases = [ (1, 0, 13), (5, -4, 9), (7, -6, 7), + ( + 12, + -9, + 4, + ), # output_chunk_length > n -> covariate requirements are capped ] - test_cases = test_cases_multi_models if mode else test_cases_one_shot - for (output_chunk_length, req_past_offset, req_future_offset) in test_cases: model = RegressionModel( lags_past_covariates=[-10], @@ -1322,7 +1318,7 @@ def test_not_enough_covariates(self): # check that given the required offsets no ValueError is raised model.predict( - 10, + n, series=target_series[:-25], past_covariates=past_covariates[: -25 + req_past_offset], future_covariates=future_covariates[: -25 + req_future_offset], @@ -1330,7 +1326,7 @@ def test_not_enough_covariates(self): # check that one less past covariate time step causes ValueError with self.assertRaises(ValueError): model.predict( - 10, + n, series=target_series[:-25], past_covariates=past_covariates[: -26 + req_past_offset], future_covariates=future_covariates[: -25 + req_future_offset], @@ -1338,7 +1334,7 @@ def test_not_enough_covariates(self): # check that one less future covariate time step causes ValueError with self.assertRaises(ValueError): model.predict( - 10, + n, series=target_series[:-25], past_covariates=past_covariates[: -25 + req_past_offset], future_covariates=future_covariates[: -26 + req_future_offset], @@ -1417,21 +1413,32 @@ def test_integer_indexed_series(self): def test_encoders(self): max_past_lag = -4 max_future_lag = 4 - n_comp_past, n_comp_future = 2, 1 - extend_past = np.random.randn(15, n_comp_past) - extend_future = np.random.randn(15, n_comp_future) - target_series = [ts[:10] for ts in self.target_series] - past_covs = [ - covs[:10].append_values(extend_past) for covs in self.past_covariates - ] - future_covs = [ - covs[:10].append_values(extend_future) for covs in self.future_covariates - ] + # target + t1 = tg.linear_timeseries( + start=pd.Timestamp("2000-01-01"), end=pd.Timestamp("2000-12-01"), freq="MS" + ) + t2 = tg.linear_timeseries( + start=pd.Timestamp("2001-01-01"), end=pd.Timestamp("2001-12-01"), freq="MS" + ) + ts = [t1, t2] + + # past and future covariates longer than target + n_comp = 2 + covs = TimeSeries.from_times_and_values( + tg.generate_index( + start=pd.Timestamp("1999-01-01"), + end=pd.Timestamp("2002-12-01"), + freq="MS", + ), + values=np.random.randn(48, n_comp), + ) + pc = [covs, covs] + fc = [covs, covs] examples = ["past", "future", "mixed"] covariates_examples = { - "past": {"past_covariates": past_covs}, - "future": {"future_covariates": future_covs}, - "mixed": {"past_covariates": past_covs, "future_covariates": future_covs}, + "past": {"past_covariates": pc}, + "future": {"future_covariates": fc}, + "mixed": {"past_covariates": pc, "future_covariates": fc}, } encoder_examples = { "past": {"datetime_attribute": {"past": ["hour"]}}, @@ -1444,81 +1451,98 @@ def test_encoders(self): multi_models_mode = [True, False] for mode in multi_models_mode: - for model_cls in [ - RegressionModel, - LinearRegressionModel, - LightGBMModel, - XGBModel, - ]: - model_pc_valid0 = model_cls( - lags=2, add_encoders=encoder_examples["past"], multi_models=mode - ) - model_fc_valid0 = model_cls( - lags=2, add_encoders=encoder_examples["future"], multi_models=mode - ) - model_mixed_valid0 = model_cls( - lags=2, add_encoders=encoder_examples["mixed"], multi_models=mode - ) - - # encoders will not generate covariates without lags - for model in [model_pc_valid0, model_fc_valid0, model_mixed_valid0]: - model.fit(target_series) - assert not model.encoders.encoding_available - _ = model.predict(n=1, series=target_series) - _ = model.predict(n=3, series=target_series) - - model_pc_valid1 = model_cls( - lags=2, - lags_past_covariates=[max_past_lag, -1], - add_encoders=encoder_examples["past"], - ) - model_fc_valid1 = model_cls( - lags=2, - lags_future_covariates=[0, max_future_lag], - add_encoders=encoder_examples["future"], - ) - model_mixed_valid1 = model_cls( - lags=2, - lags_past_covariates=[max_past_lag, -1], - lags_future_covariates=[0, max_future_lag], - add_encoders=encoder_examples["mixed"], - ) - - for model, ex in zip( - [model_pc_valid1, model_fc_valid1, model_mixed_valid1], examples - ): - covariates = covariates_examples[ex] - # don't pass covariates, let them be generated by encoders. Test single target series input - model_copy = copy.deepcopy(model) - model_copy.fit(target_series[0]) - assert model_copy.encoders.encoding_available - self.helper_test_encoders_settings(model_copy, ex) - - _ = model_copy.predict(n=1, series=target_series) - self.helper_compare_encoded_covs_with_ref( - model_copy, target_series, covariates, n=1 + for ocl in [1, 2]: + for model_cls in [ + RegressionModel, + LinearRegressionModel, + LightGBMModel, + XGBModel, + ]: + model_pc_valid0 = model_cls( + lags=2, + add_encoders=encoder_examples["past"], + multi_models=mode, + output_chunk_length=ocl, ) - - _ = model_copy.predict(n=3, series=target_series) - self.helper_compare_encoded_covs_with_ref( - model_copy, target_series, covariates, n=3 + model_fc_valid0 = model_cls( + lags=2, + add_encoders=encoder_examples["future"], + multi_models=mode, + output_chunk_length=ocl, + ) + model_mixed_valid0 = model_cls( + lags=2, + add_encoders=encoder_examples["mixed"], + multi_models=mode, + output_chunk_length=ocl, ) - _ = model_copy.predict(n=8, series=target_series) - self.helper_compare_encoded_covs_with_ref( - model_copy, target_series, covariates, n=8 + # encoders will not generate covariates without lags + for model in [model_pc_valid0, model_fc_valid0, model_mixed_valid0]: + model.fit(ts) + assert not model.encoders.encoding_available + _ = model.predict(n=1, series=ts) + _ = model.predict(n=3, series=ts) + + model_pc_valid1 = model_cls( + lags=2, + lags_past_covariates=[max_past_lag, -1], + add_encoders=encoder_examples["past"], + multi_models=mode, + output_chunk_length=ocl, + ) + model_fc_valid1 = model_cls( + lags=2, + lags_future_covariates=[0, max_future_lag], + add_encoders=encoder_examples["future"], + multi_models=mode, + output_chunk_length=ocl, ) + model_mixed_valid1 = model_cls( + lags=2, + lags_past_covariates=[max_past_lag, -1], + lags_future_covariates=[0, max_future_lag], + add_encoders=encoder_examples["mixed"], + multi_models=mode, + output_chunk_length=ocl, + ) + + for model, ex in zip( + [model_pc_valid1, model_fc_valid1, model_mixed_valid1], examples + ): + covariates = covariates_examples[ex] + # don't pass covariates, let them be generated by encoders. Test single target series input + model_copy = copy.deepcopy(model) + model_copy.fit(ts[0]) + assert model_copy.encoders.encoding_available + self.helper_test_encoders_settings(model_copy, ex) + _ = model_copy.predict(n=1, series=ts) + self.helper_compare_encoded_covs_with_ref( + model_copy, ts, covariates, n=1, ocl=ocl, multi_model=mode + ) + + _ = model_copy.predict(n=3, series=ts) + self.helper_compare_encoded_covs_with_ref( + model_copy, ts, covariates, n=3, ocl=ocl, multi_model=mode + ) + + _ = model_copy.predict(n=8, series=ts) + self.helper_compare_encoded_covs_with_ref( + model_copy, ts, covariates, n=8, ocl=ocl, multi_model=mode + ) - # manually pass covariates, let encoders add more - model.fit(target_series, **covariates) - assert model.encoders.encoding_available - self.helper_test_encoders_settings(model, ex) - _ = model.predict(n=1, series=target_series, **covariates) - _ = model.predict(n=3, series=target_series, **covariates) - _ = model.predict(n=8, series=target_series, **covariates) + # manually pass covariates, let encoders add more + model.fit(ts, **covariates) + assert model.encoders.encoding_available + self.helper_test_encoders_settings(model, ex) + _ = model.predict(n=1, series=ts, **covariates) + _ = model.predict(n=3, series=ts, **covariates) + _ = model.predict(n=8, series=ts, **covariates) @staticmethod - def helper_compare_encoded_covs_with_ref(model, target_series, covariates, n): + def helper_compare_encoded_covs_with_ref( + model, ts, covariates, n, ocl, multi_model + ): """checks that covariates generated by encoders fulfill the requirements compared to some reference covariates: What has to match: @@ -1527,47 +1551,56 @@ def helper_compare_encoded_covs_with_ref(model, target_series, covariates, n): - generated/encoded covariates at training time must have the same start time as reference - generated/encoded covariates at prediction time must have the same end time as reference """ + + def generate_expected_times(ts, n_predict=0) -> dict: + """generates expected start and end times for the corresponding covariates.""" + freq = ts[0].freq + + def to_ts(dt): + return pd.Timestamp(dt, freq=freq) + + def train_start_end(start_base, end_base): + start = to_ts(start_base) - int(not multi_model) * (ocl - 1) * freq + if not n_predict: + end = to_ts(end_base) - (ocl - 1) * freq + else: + end = to_ts(end_base) + freq * max(n_predict - ocl, 0) + return start, end + + if not n_predict: + # expected train start, and end + pc1_start, pc1_end = train_start_end("1999-11-01", "2000-11-01") + pc2_start, pc2_end = train_start_end("2000-11-01", "2001-11-01") + fc1_start, fc1_end = train_start_end("2000-03-01", "2001-04-01") + fc2_start, fc2_end = train_start_end("2001-03-01", "2002-04-01") + else: + # expected inference start, and end + pc1_start, pc1_end = train_start_end("2000-09-01", "2000-12-01") + pc2_start, pc2_end = train_start_end("2001-09-01", "2001-12-01") + fc1_start, fc1_end = train_start_end("2001-01-01", "2001-05-01") + fc2_start, fc2_end = train_start_end("2002-01-01", "2002-05-01") + + times = { + "pc_start": [pc1_start, pc2_start], + "pc_end": [pc1_end, pc2_end], + "fc_start": [fc1_start, fc2_start], + "fc_end": [fc1_end, fc2_end], + } + return times + covs_reference = ( covariates.get("past_covariates"), covariates.get("future_covariates"), ) - covs_generated_train = model.encoders.encode_train(target=target_series) - covs_generated_infer = model.encoders.encode_inference( - n=model._get_encoders_n(n), target=target_series - ) + covs_generated_train = model.encoders.encode_train(target=ts) + covs_generated_infer = model.encoders.encode_inference(n=n, target=ts) refer_past, refer_future = covs_reference[0], covs_reference[1] train_past, train_future = covs_generated_train[0], covs_generated_train[1] infer_past, infer_future = covs_generated_infer[0], covs_generated_infer[1] - def generate_expected_times(model, target_series, n=0) -> dict: - """generates expected start and end times for the corresponding covariates.""" - times = {"pc_start": [], "pc_end": [], "fc_start": [], "fc_end": []} - max_past_lag = abs(min(min(model.lags.get("past", [0])), 0)) - max_future_lag = max(max(model.lags.get("future", [0])), 0) - for ts in target_series: - if not n: - times["pc_start"].append(ts.start_time()) - times["pc_end"].append(ts.end_time()) - times["fc_start"].append(ts.start_time()) - times["fc_end"].append(ts.end_time()) - else: - # -1 as last step is inclusive - times["pc_start"].append( - ts.end_time() - ts.freq * (max_past_lag - 1) - ) - times["pc_end"].append(ts.end_time() + ts.freq * (n - 1)) - times["fc_start"].append( - ts.end_time() - ts.freq * (max_past_lag - 1) - ) - times["fc_end"].append( - ts.end_time() + ts.freq * (n + max_future_lag) - ) - return times - - t_train = generate_expected_times(model, target_series) - t_infer = generate_expected_times(model, target_series, n=n) - + t_train = generate_expected_times(ts) + t_infer = generate_expected_times(ts, n_predict=n) if train_past is None: assert infer_past is None and refer_past is None else: