feature(KTP-767): Implemented XGBoost Multioutput Arctan Loss

README.md

@@ -91,4 +91,4 @@ Please read [CODE_OF_CONDUCT.md](https://github.com/OpenSTEF/.github/blob/main/C
 
 # Contact
 Please read [SUPPORT.md](https://github.com/OpenSTEF/.github/blob/main/SUPPORT.md) for how to connect and get into contact with the OpenSTEF project
-
+ 

openstef/data_classes/prediction_job.py

@@ -26,6 +26,7 @@ class PredictionJobDataClass(BaseModel):
         - ``"lgb"``
         - ``"linear"``
         - ``"linear_quantile"``
+        - ``"xgb_multioutput_quantile"``
 
     If unsure what to pick, choose ``"xgb"``.
 

openstef/enums.py

@@ -8,6 +8,7 @@
 class MLModelType(Enum):
     XGB = "xgb"
     XGB_QUANTILE = "xgb_quantile"
+    XGB_MULTIOUTPUT_QUANTILE = "xgb_multioutput_quantile"
     LGB = "lgb"
     LINEAR = "linear"
     LINEAR_QUANTILE = "linear_quantile"

openstef/metrics/metrics.py

@@ -431,3 +431,54 @@ def xgb_quantile_obj(
     hess = np.ones_like(preds)
 
     return grad, hess
+
+
+def arctan_loss(y_true, y_pred, taus, s=0.1):
+    """Compute the arctan pinball loss.
+
+    Note that XGBoost outputs the predictions in a slightly peculiar manner.
+    Suppose we have 100 data points and we predict 10 quantiles. The predictions
+    will be an array of size (1000 x 1). We first resize this to a (100x10) array
+    where each row corresponds to the 10 predicted quantile for a single data
+    point. We then use a for-loop (over the 10 columns) to calculate the gradients
+    and second derivatives. Legibility was chosen over efficiency. This part
+    can be made more efficient.
+
+    Args:
+        y_true: An array containing the true observations.
+        y_pred: An array containing the predicted quantiles.
+        taus: A list containing the true desired coverage of the quantiles.
+        s: A smoothing parameter.
+
+    Returns:
+        grad: An array containing the (negative) gradients with respect to y_pred.
+        hess: An array containing the second derivative with respect to y_pred.
+
+    """
+    size = len(y_true)
+    n_dim = len(taus)  # The number of columns
+    n_rows = size // n_dim
+
+    # Resize the predictions and targets.
+    # Each column corresponds to a quantile, each row to a data point.
+    y_pred = np.reshape(y_pred, (n_rows, n_dim))
+    y_true = np.reshape(y_true, (n_rows, n_dim))
+
+    # Calculate the differences
+    u = y_true - y_pred
+
+    # Calculate the gradient and second derivatives
+    grad = np.zeros_like(y_pred)
+    hess = np.zeros_like(y_pred)
+    z = u / s
+    for i, tau in enumerate(taus):
+        x = 1 + z[:, i] ** 2
+        grad[:, i] = (
+            tau - 0.5 + 1 / np.pi * np.arctan(z[:, i]) + z[:, i] / (np.pi) * x**-1
+        )
+        hess[:, i] = 2 / (np.pi * s) * x ** (-2)
+
+    # Reshape back to the original shape.
+    grad = grad.reshape(size)
+    hess = hess.reshape(size)
+    return -grad / n_dim, hess / n_dim

openstef/model/model_creator.py

@@ -15,6 +15,9 @@
 from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.model.regressors.xgb import XGBOpenstfRegressor
 from openstef.model.regressors.xgb_quantile import XGBQuantileOpenstfRegressor
+from openstef.model.regressors.xgb_multioutput_quantile import (
+    XGBMultiOutputQuantileOpenstfRegressor,
+)
 from openstef.settings import Settings
 
 structlog.configure(
@@ -87,6 +90,16 @@
         "max_depth",
         "early_stopping_rounds",
     ],
+    MLModelType.XGB_MULTIOUTPUT_QUANTILE: [
+        "quantiles",
+        "gamma",
+        "colsample_bytree",
+        "subsample",
+        "min_child_weight",
+        "max_depth",
+        "early_stopping_rounds",
+        "arctan_smoothing",
+    ],
     MLModelType.LINEAR: [
         "missing_values",
         "imputation_strategy",
@@ -117,6 +130,7 @@ class ModelCreator:
         MLModelType.XGB: XGBOpenstfRegressor,
         MLModelType.LGB: LGBMOpenstfRegressor,
         MLModelType.XGB_QUANTILE: XGBQuantileOpenstfRegressor,
+        MLModelType.XGB_MULTIOUTPUT_QUANTILE: XGBMultiOutputQuantileOpenstfRegressor,
         MLModelType.LINEAR: LinearOpenstfRegressor,
         MLModelType.LINEAR_QUANTILE: LinearQuantileOpenstfRegressor,
         MLModelType.ARIMA: ARIMAOpenstfRegressor,

openstef/model/objective.py

@@ -349,6 +349,36 @@ def get_pruning_callback(self, trial: optuna.trial.FrozenTrial):
         )
 
 
+class XGBMultioutputQuantileRegressorObjective(RegressorObjective):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.model_type = MLModelType.XGB_QUANTILE
+
+    def get_params(self, trial: optuna.trial.FrozenTrial) -> dict:
+        """Get parameters for XGB Multioutput Quantile Regressor Objective with objective specific parameters.
+
+        Args: trial
+
+        Returns:
+            Dictionary with hyperparameter name as key and hyperparamer value as value.
+
+        """
+        # Filtered default parameters
+        model_params = super().get_params(trial)
+
+        # XGB specific parameters
+        params = {
+            "gamma": trial.suggest_float("gamma", 1e-8, 1.0),
+            "arctan_smoothing": trial.suggest_float("arctan_smoothing", 0.025, 0.15),
+        }
+        return {**model_params, **params}
+
+    def get_pruning_callback(self, trial: optuna.trial.FrozenTrial):
+        return optuna.integration.XGBoostPruningCallback(
+            trial, observation_key=f"validation_1-{self.eval_metric}"
+        )
+
+
 class LinearRegressorObjective(RegressorObjective):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

openstef/model/objective_creator.py

@@ -12,6 +12,7 @@
     RegressorObjective,
     XGBQuantileRegressorObjective,
     XGBRegressorObjective,
+    XGBMultioutputQuantileRegressorObjective,
 )
 from openstef.model.regressors.custom_regressor import (
     create_custom_objective,
@@ -24,7 +25,9 @@ class ObjectiveCreator:
         MLModelType.XGB: XGBRegressorObjective,
         MLModelType.LGB: LGBRegressorObjective,
         MLModelType.XGB_QUANTILE: XGBQuantileRegressorObjective,
+        MLModelType.XGB_MULTIOUTPUT_QUANTILE: XGBMultioutputQuantileRegressorObjective,
         MLModelType.LINEAR: LinearRegressorObjective,
+        MLModelType.LINEAR_QUANTILE: LinearRegressorObjective,
         MLModelType.ARIMA: ARIMARegressorObjective,
     }