diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ed706cfd..aa2bf4d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add seasonal_plot ([#628](https://github.com/tinkoff-ai/etna/pull/628)) - - Add plot_periodogram ([#606](https://github.com/tinkoff-ai/etna/pull/606)) -- +- Add support of quantiles in backtest ([#652](https://github.com/tinkoff-ai/etna/pull/652)) - Fixed bug in SARIMAX model with `horizon`=1 ([#637](https://github.com/tinkoff-ai/etna/pull/637)) - Add prediction_actual_scatter_plot ([#610](https://github.com/tinkoff-ai/etna/pull/610)) - Add plot_holidays ([#624](https://github.com/tinkoff-ai/etna/pull/624)) diff --git a/etna/pipeline/base.py b/etna/pipeline/base.py index bbc3889d2..207c0782a 100644 --- a/etna/pipeline/base.py +++ b/etna/pipeline/base.py @@ -161,6 +161,7 @@ def backtest( aggregate_metrics: bool = False, n_jobs: int = 1, joblib_params: Optional[Dict[str, Any]] = None, + forecast_params: Optional[Dict[str, Any]] = None, ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: """Run backtest with the pipeline. @@ -180,6 +181,8 @@ def backtest( Number of jobs to run in parallel joblib_params: Additional parameters for :py:class:`joblib.Parallel` + forecast_params: + Additional parameters for :py:func:`~etna.pipeline.base.BasePipeline.forecast` Returns ------- @@ -373,14 +376,20 @@ def _compute_metrics(metrics: List[Metric], y_true: TSDataset, y_pred: TSDataset return metrics_values def _run_fold( - self, train: TSDataset, test: TSDataset, fold_number: int, mask: FoldMask, metrics: List[Metric] + self, + train: TSDataset, + test: TSDataset, + fold_number: int, + mask: FoldMask, + metrics: List[Metric], + forecast_params: Dict[str, Any], ) -> Dict[str, Any]: """Run fit-forecast pipeline of model for one fold.""" tslogger.start_experiment(job_type="crossval", group=str(fold_number)) pipeline = deepcopy(self) pipeline.fit(ts=train) - forecast = pipeline.forecast() + forecast = pipeline.forecast(**forecast_params) fold: Dict[str, Any] = {} for stage_name, stage_df in zip(("train", "test"), (train, test)): fold[f"{stage_name}_timerange"] = {} @@ -471,6 +480,7 @@ def backtest( aggregate_metrics: bool = False, n_jobs: int = 1, joblib_params: Optional[Dict[str, Any]] = None, + forecast_params: Optional[Dict[str, Any]] = None, ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: """Run backtest with the pipeline. @@ -490,6 +500,8 @@ def backtest( Number of jobs to run in parallel joblib_params: Additional parameters for :py:class:`joblib.Parallel` + forecast_params: + Additional parameters for :py:func:`~etna.pipeline.base.BasePipeline.forecast` Returns ------- @@ -499,13 +511,21 @@ def backtest( if joblib_params is None: joblib_params = dict(verbose=11, backend="multiprocessing", mmap_mode="c") + if forecast_params is None: + forecast_params = dict() + self._init_backtest() self._validate_backtest_metrics(metrics=metrics) masks = self._prepare_fold_masks(ts=ts, masks=n_folds, mode=mode) folds = Parallel(n_jobs=n_jobs, **joblib_params)( delayed(self._run_fold)( - train=train, test=test, fold_number=fold_number, mask=masks[fold_number], metrics=metrics + train=train, + test=test, + fold_number=fold_number, + mask=masks[fold_number], + metrics=metrics, + forecast_params=forecast_params, ) for fold_number, (train, test) in enumerate( self._generate_folds_datasets(ts=ts, masks=masks, horizon=self.horizon) diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index 98da11519..79eb9f310 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -13,6 +13,7 @@ from etna.metrics import SMAPE from etna.metrics import Metric from etna.metrics import MetricAggregationMode +from etna.metrics import Width from etna.models import LinearPerSegmentModel from etna.models import MovingAverageModel from etna.models import NaiveModel @@ -450,7 +451,7 @@ def test_run_fold(ts_run_fold: TSDataset, mask: FoldMask, expected: Dict[str, Li ) pipeline = Pipeline(model=NaiveModel(lag=5), transforms=[], horizon=4) - fold = pipeline._run_fold(train, test, 1, mask, [MAE()]) + fold = pipeline._run_fold(train, test, 1, mask, [MAE()], forecast_params=dict()) for seg in fold["metrics"]["MAE"].keys(): assert fold["metrics"]["MAE"][seg] == expected[seg] @@ -487,3 +488,17 @@ def test_backtest_two_points(masked_ts: TSDataset, lag: int, expected: Dict[str, for segment in expected.keys(): assert segment in metrics.keys() np.testing.assert_array_almost_equal(expected[segment], metrics[segment]) + + +def test_sanity_backtest_naive_with_intervals(weekly_period_ts): + train_ts, _ = weekly_period_ts + quantiles = (0.01, 0.99) + pipeline = Pipeline(model=NaiveModel(), horizon=5) + _, forecast_df, _ = pipeline.backtest( + ts=train_ts, + metrics=[MAE(), Width(quantiles=quantiles)], + forecast_params={"quantiles": quantiles, "prediction_interval": True}, + ) + features = forecast_df.columns.get_level_values(1) + assert f"target_{quantiles[0]}" in features + assert f"target_{quantiles[1]}" in features