Add support of quantiles in backtest

CHANGELOG.md

@@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add seasonal_plot ([#628](https://github.com/tinkoff-ai/etna/pull/628))
 - 
 - Add plot_periodogram ([#606](https://github.com/tinkoff-ai/etna/pull/606))
-- 
+- Add support of quantiles in backtest ([#652](https://github.com/tinkoff-ai/etna/pull/652))
 - Fixed bug in SARIMAX model with `horizon`=1 ([#637](https://github.com/tinkoff-ai/etna/pull/637))
 - Add prediction_actual_scatter_plot ([#610](https://github.com/tinkoff-ai/etna/pull/610))
 - Add plot_holidays ([#624](https://github.com/tinkoff-ai/etna/pull/624))

etna/pipeline/base.py

@@ -161,6 +161,7 @@ def backtest(
         aggregate_metrics: bool = False,
         n_jobs: int = 1,
         joblib_params: Optional[Dict[str, Any]] = None,
+        forecast_params: Optional[Dict[str, Any]] = None,
     ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
         """Run backtest with the pipeline.
 
@@ -373,14 +374,20 @@ def _compute_metrics(metrics: List[Metric], y_true: TSDataset, y_pred: TSDataset
         return metrics_values
 
     def _run_fold(
-        self, train: TSDataset, test: TSDataset, fold_number: int, mask: FoldMask, metrics: List[Metric]
+        self,
+        train: TSDataset,
+        test: TSDataset,
+        fold_number: int,
+        mask: FoldMask,
+        metrics: List[Metric],
+        forecast_params: Dict[str, Any],
     ) -> Dict[str, Any]:
         """Run fit-forecast pipeline of model for one fold."""
         tslogger.start_experiment(job_type="crossval", group=str(fold_number))
 
         pipeline = deepcopy(self)
         pipeline.fit(ts=train)
-        forecast = pipeline.forecast()
+        forecast = pipeline.forecast(**forecast_params)
         fold: Dict[str, Any] = {}
         for stage_name, stage_df in zip(("train", "test"), (train, test)):
             fold[f"{stage_name}_timerange"] = {}
@@ -471,6 +478,7 @@ def backtest(
         aggregate_metrics: bool = False,
         n_jobs: int = 1,
         joblib_params: Optional[Dict[str, Any]] = None,
+        forecast_params: Optional[Dict[str, Any]] = None,
     ) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
         """Run backtest with the pipeline.
 
@@ -490,6 +498,8 @@ def backtest(
             Number of jobs to run in parallel
         joblib_params:
             Additional parameters for :py:class:`joblib.Parallel`
+        forecast_params:
+            Additional parameters for :py:func:`~etna.pipeline.base.BasePipeline.forecast`
 
         Returns
         -------
@@ -499,13 +509,21 @@ def backtest(
         if joblib_params is None:
             joblib_params = dict(verbose=11, backend="multiprocessing", mmap_mode="c")
 
+        if forecast_params is None:
+            forecast_params = dict()
+
         self._init_backtest()
         self._validate_backtest_metrics(metrics=metrics)
         masks = self._prepare_fold_masks(ts=ts, masks=n_folds, mode=mode)
 
         folds = Parallel(n_jobs=n_jobs, **joblib_params)(
             delayed(self._run_fold)(
-                train=train, test=test, fold_number=fold_number, mask=masks[fold_number], metrics=metrics
+                train=train,
+                test=test,
+                fold_number=fold_number,
+                mask=masks[fold_number],
+                metrics=metrics,
+                forecast_params=forecast_params,
             )
             for fold_number, (train, test) in enumerate(
                 self._generate_folds_datasets(ts=ts, masks=masks, horizon=self.horizon)

tests/test_pipeline/test_pipeline.py

@@ -13,6 +13,7 @@
 from etna.metrics import SMAPE
 from etna.metrics import Metric
 from etna.metrics import MetricAggregationMode
+from etna.metrics import Width
 from etna.models import LinearPerSegmentModel
 from etna.models import MovingAverageModel
 from etna.models import NaiveModel
@@ -450,7 +451,7 @@ def test_run_fold(ts_run_fold: TSDataset, mask: FoldMask, expected: Dict[str, Li
     )
 
     pipeline = Pipeline(model=NaiveModel(lag=5), transforms=[], horizon=4)
-    fold = pipeline._run_fold(train, test, 1, mask, [MAE()])
+    fold = pipeline._run_fold(train, test, 1, mask, [MAE()], forecast_params=dict())
     for seg in fold["metrics"]["MAE"].keys():
         assert fold["metrics"]["MAE"][seg] == expected[seg]
 
@@ -487,3 +488,17 @@ def test_backtest_two_points(masked_ts: TSDataset, lag: int, expected: Dict[str,
     for segment in expected.keys():
         assert segment in metrics.keys()
         np.testing.assert_array_almost_equal(expected[segment], metrics[segment])
+
+
+def test_sanity_backtest_naive_with_intervals(weekly_period_ts):
+    train_ts, _ = weekly_period_ts
+    quantiles = (0.01, 0.99)
+    pipeline = Pipeline(model=NaiveModel(), horizon=5)
+    _, forecast_df, _ = pipeline.backtest(
+        ts=train_ts,
+        metrics=[MAE(), Width(quantiles=quantiles)],
+        forecast_params={"quantiles": quantiles, "prediction_interval": True},
+    )
+    features = forecast_df.columns.get_level_values(1)
+    assert f"target_{quantiles[0]}" in features
+    assert f"target_{quantiles[1]}" in features