Remove cost error from the benchmark

.github/ISSUE_TEMPLATE/new_problem.md

@@ -8,8 +8,7 @@ assignees: ''
 
 ### Problem
 
-I propose to add the following problem to the *GitHub free-for-all* test set.
-The problem can be built and tested as follows:
+I propose to add the following problem to the *GitHub free-for-all* test set. The problem can be built and tested as follows:
 
 ```python
 import numpy as np
@@ -48,16 +47,26 @@ if __name__ == "__main__":
 
 This problem is interesting because...
 
-### Solution and optimal cost
+### Solution
 
 <!--
-    If you know a formula for the solution of the problem, or the optimal cost,
-    write them down here. This is not a requirement but it can help us debug
-    solver outputs later on.
+    If you know a formula for the solution of the problem, you can write it
+    down here. This is not a requirement but it can help us debug solver
+    outputs later on.
 -->
 
-- Solution: $x^\* = ...$
-- Optimal cost: $\frac12 x^{\*T} P x^\* + q^T x^\* = ...$
+The solution to this problem is:
+
+```math
+\begin{align*}
+x^\* & = ... \\
+y^\* & = ... \\
+z^\* & = ... \\
+z_{\mathit{box}}^\* & = ... \\
+\end{align*}
+```
+
+where $x^\*$ is the primal vector, $y^\*$ the dual vector for equality constraints, $z^\*$ the dual vector for inequality constraints, and $z_{\mathit{box}}^\*$ the dual vector for box constraints.
 
 ### References
 

README.md

@@ -69,7 +69,6 @@ We evaluate QP solvers based on the following metrics:
     - **Primal residual:** maximum error on equality and inequality constraints at the returned solution.
     - **Dual residual:** maximum error on the dual feasibility condition at the returned solution.
     - **Duality gap:** value of the duality gap at the returned solution.
-- **Cost error:** difference between the solution cost and the known optimal cost.
 
 ### Shifted geometric mean
 

github_ffa/github_ffa.py

@@ -76,28 +76,24 @@ def define_tolerances(self) -> None:
         """Define test set tolerances."""
         self.tolerances = {
             "default": Tolerance(
-                cost=1000.0,
                 primal=1.0,
                 dual=1.0,
                 gap=1.0,
                 runtime=100.0,
             ),
             "high_accuracy": Tolerance(
-                cost=1000.0,
                 primal=1e-9,
                 dual=1e-9,
                 gap=1e-9,
                 runtime=100.0,
             ),
             "low_accuracy": Tolerance(
-                cost=1000.0,
                 primal=1e-3,
                 dual=1e-3,
                 gap=1e-3,
                 runtime=100.0,
             ),
             "mid_accuracy": Tolerance(
-                cost=1000.0,
                 primal=1e-6,
                 dual=1e-6,
                 gap=1e-6,

github_ffa/problems/ghffa01.py

@@ -44,7 +44,6 @@ def get_problem(alpha: float):
         lb=None,
         ub=None,
         name=f"GHFFA01_{alpha=}",
-        optimal_cost=0.5 / (1 + alpha**2),
     )
 
 

github_ffa/problems/ghffa02.py

@@ -42,7 +42,6 @@ def get_problem(alpha: float):
         lb=None,
         ub=None,
         name=f"GHFFA02_{alpha=}",
-        optimal_cost=0.5 / alpha**2,
     )
 
 

github_ffa/problems/ghffa03.py

@@ -45,7 +45,6 @@ def get_problem(n: int):
         lb=None,
         ub=None,
         name=f"GHFFA03_{n=}",
-        optimal_cost=0.0,
     )
 
 

maros_meszaros/maros_meszaros.py

@@ -55,28 +55,24 @@ def define_tolerances(self) -> None:
         """Define test set tolerances."""
         self.tolerances = {
             "default": Tolerance(
-                cost=1000.0,
                 primal=1.0,
                 dual=1.0,
                 gap=1.0,
                 runtime=1000.0,
             ),
             "high_accuracy": Tolerance(
-                cost=1000.0,
                 primal=1e-9,
                 dual=1e-9,
                 gap=1e-9,
                 runtime=1000.0,
             ),
             "low_accuracy": Tolerance(
-                cost=1000.0,
                 primal=1e-3,
                 dual=1e-3,
                 gap=1e-3,
                 runtime=1000.0,
             ),
             "mid_accuracy": Tolerance(
-                cost=1000.0,
                 primal=1e-6,
                 dual=1e-6,
                 gap=1e-6,

qpbenchmark/problem.py

@@ -28,15 +28,10 @@ class Problem(qpsolvers.Problem):
     """Quadratic program.
 
     Attributes:
-        cost_offset: Cost offset, used to compare solution cost to a known
-            optimal one. Defaults to zero.
         name: Name of the problem, for reporting.
-        optimal_cost: If known, cost at the optimum of the problem.
     """
 
-    cost_offset: float
     name: str
-    optimal_cost: Optional[float]
 
     def __init__(
         self,
@@ -49,14 +44,10 @@ def __init__(
         lb: Optional[np.ndarray],
         ub: Optional[np.ndarray],
         name: str,
-        optimal_cost: Optional[float] = None,
-        cost_offset: float = 0.0,
     ):
         """Quadratic program in qpsolvers format."""
         super().__init__(P, q, G, h, A, b, lb, ub)
-        self.cost_offset = cost_offset
         self.name = name
-        self.optimal_cost = optimal_cost
 
     def to_dense(self):
         """Return dense version.
@@ -74,8 +65,6 @@ def to_dense(self):
             self.lb,
             self.ub,
             name=self.name,
-            optimal_cost=self.optimal_cost,
-            cost_offset=self.cost_offset,
         )
 
     def to_sparse(self):
@@ -95,26 +84,4 @@ def to_sparse(self):
             self.lb,
             self.ub,
             name=self.name,
-            optimal_cost=self.optimal_cost,
-            cost_offset=self.cost_offset,
         )
-
-    def cost_error(self, solution: qpsolvers.Solution) -> Optional[float]:
-        """Compute difference between found cost and the optimal one.
-
-        Args:
-            solution: Problem solution.
-
-        Returns:
-            Cost error, i.e. deviation from the (known) optimal cost.
-
-        Note:
-            Cost errors can be negative when the primal residual is large. We
-            count that as errors as well using absolute values.
-        """
-        x = solution.x
-        if not solution.found or x is None or self.optimal_cost is None:
-            return None
-        P, q = self.P, self.q
-        cost = 0.5 * x.dot(P.dot(x)) + q.dot(x) + self.cost_offset
-        return abs(cost - self.optimal_cost)

qpbenchmark/report.py

@@ -51,7 +51,6 @@ class Report:
     # Reports are big and linear, thus with many instance attributes.
 
     __correct_rate_df: pandas.DataFrame
-    __cost_df: pandas.DataFrame
     __dual_df: pandas.DataFrame
     __gap_df: pandas.DataFrame
     __primal_df: pandas.DataFrame
@@ -70,7 +69,6 @@ def __init__(self, author: str, results: Results):
             results: Results from which the report should be generated.
         """
         self.__correct_rate_df = pandas.DataFrame()
-        self.__cost_df = pandas.DataFrame()
         self.__dual_df = pandas.DataFrame()
         self.__gap_df = pandas.DataFrame()
         self.__primal_df = pandas.DataFrame()
@@ -92,7 +90,7 @@ def get_tolerances_table(self) -> str:
             [],
             columns=["tolerance"] + names,
         )
-        tolerances = ["cost", "primal", "dual", "gap", "runtime"]
+        tolerances = ["primal", "dual", "gap", "runtime"]
         for tolerance in tolerances:
             row = {
                 "tolerance": [f"``{tolerance}``"],
@@ -198,10 +196,6 @@ def __compute_dataframes(self) -> None:
             name: tolerance.gap
             for name, tolerance in self.test_set.tolerances.items()
         }
-        cost_tolerances = {
-            name: tolerance.cost
-            for name, tolerance in self.test_set.tolerances.items()
-        }
         runtime_tolerances = {
             name: tolerance.runtime
             for name, tolerance in self.test_set.tolerances.items()
@@ -210,13 +204,11 @@ def __compute_dataframes(self) -> None:
             primal_tolerances,
             dual_tolerances,
             gap_tolerances,
-            cost_tolerances,
         )
         self.__correct_rate_df = self.results.build_correct_rate_df(
             primal_tolerances,
             dual_tolerances,
             gap_tolerances,
-            cost_tolerances,
         )
         self.__runtime_df = self.results.build_shgeom_df(
             metric="runtime",
@@ -238,11 +230,6 @@ def __compute_dataframes(self) -> None:
             shift=10.0,
             not_found_values=gap_tolerances,
         )
-        self.__cost_df = self.results.build_shgeom_df(
-            metric="cost_error",
-            shift=10.0,
-            not_found_values=cost_tolerances,
-        )
 
     def write(self, path: str) -> None:
         """Write report to a given path.
@@ -317,8 +304,7 @@ def __write_toc(self, fh: io.TextIOWrapper) -> None:
     * [Optimality conditions](#optimality-conditions)
         * [Primal residual](#primal-residual)
         * [Dual residual](#dual-residual)
-        * [Duality gap](#duality-gap)
-    * [Cost error](#cost-error)\n\n"""
+        * [Duality gap](#duality-gap)\n\n"""
         )
 
     def __write_description(self, fh: io.TextIOWrapper) -> None:
@@ -415,7 +401,6 @@ def __write_results_by_settings(self, fh: io.TextIOWrapper) -> None:
                     settings
                 ],
                 "[Duality gap](#duality-gap) (shm)": self.__gap_df[settings],
-                "[Cost error](#cost-error) (shm)": self.__cost_df[settings],
             }
             df = pandas.DataFrame([], index=self.__gap_df.index).assign(**cols)
             repo = "https://github.com/qpsolvers/qpbenchmark"
@@ -586,35 +571,5 @@ def __write_results_by_metric(self, fh: io.TextIOWrapper) -> None:
             "[gap tolerance](#settings)."
         )
 
-        fh.write(f"{duality_gap_table_desc}\n\n")
-        fh.write("### Cost error\n\n")
-
-        cost_error_shm_desc = (
-            "The cost error measures the difference between the known "
-            "optimal objective and the objective at the solution returned "
-            "by a solver. We use the shifted geometric mean to compare "
-            "solver cost errors over the whole test set. Intuitively, "
-            "a solver with a shifted-geometric-mean cost error of Y is "
-            "Y times less precise on the optimal cost than the best solver "
-            "over the test set. "
-            f"See [Metrics]({repo}#metrics) for details."
-        )
-
-        fh.write(f"{cost_error_shm_desc}\n\n")
-        fh.write(
-            "Shifted geometric means of solver cost errors "
-            "(1.0 is the best):\n\n"
-        )
-        fh.write(
-            f'{self.__cost_df.to_markdown(index=True, floatfmt=".1f")}\n\n'
-        )
-
-        cost_error_table_desc = (
-            "Rows are solvers and columns are solver settings. "
-            "The shift is $sh = 10$. A solver that fails to find a "
-            "solution receives a cost error equal to the "
-            "[cost tolerance](#settings)."
-        )
-
-        fh.write(f"{cost_error_table_desc}")
+        fh.write(f"{duality_gap_table_desc}")
         fh.write("\n")  # newline at end of file

qpbenchmark/results.py

@@ -72,7 +72,6 @@ def __init__(self, csv_path: str, test_set: TestSet):
                 "primal_residual",
                 "dual_residual",
                 "duality_gap",
-                "cost_error",
             ],
         ).astype(
             {
@@ -84,7 +83,6 @@ def __init__(self, csv_path: str, test_set: TestSet):
                 "primal_residual": float,
                 "dual_residual": float,
                 "duality_gap": float,
-                "cost_error": float,
             }
         )
         if os.path.exists(csv_path):
@@ -167,7 +165,6 @@ def update(
                         "primal_residual": [solution.primal_residual()],
                         "dual_residual": [solution.dual_residual()],
                         "duality_gap": [solution.duality_gap()],
-                        "cost_error": [problem.cost_error(solution)],
                     }
                 ),
             ],
@@ -179,15 +176,13 @@ def build_success_rate_df(
         primal_tolerances: Dict[str, float],
         dual_tolerances: Dict[str, float],
         gap_tolerances: Dict[str, float],
-        cost_tolerances: Dict[str, float],
     ) -> Tuple[pandas.DataFrame, pandas.DataFrame]:
         """Build the success-rate data frame.
 
         Args:
             primal_tolerances: Primal-residual tolerance for each settings.
             dual_tolerances: Dual-residual tolerance for each settings.
             gap_tolerances: Duality-gap tolerance for each settings.
-            cost_tolerances: Cost tolerance for each settings.
 
         Returns:
             Success-rate data frames.
@@ -200,7 +195,6 @@ def build_success_rate_df(
             & (df["primal_residual"] < primal_tolerances[settings])
             & (df["dual_residual"] < dual_tolerances[settings])
             & (df["duality_gap"] < gap_tolerances[settings])
-            & (df["cost_error"].abs() < cost_tolerances[settings])
             for settings in all_settings
         }
         success_rate_df = (
@@ -229,15 +223,13 @@ def build_correct_rate_df(
         primal_tolerances: Dict[str, float],
         dual_tolerances: Dict[str, float],
         gap_tolerances: Dict[str, float],
-        cost_tolerances: Dict[str, float],
     ) -> Tuple[pandas.DataFrame, pandas.DataFrame]:
         """Build the correctness-rate data frame.
 
         Args:
             primal_tolerances: Primal-residual tolerance for each settings.
             dual_tolerances: Dual-residual tolerance for each settings.
             gap_tolerances: Duality-gap tolerance for each settings.
-            cost_tolerances: Cost tolerance for each settings.
 
         Returns:
             Correctness-rate data frames.
@@ -250,7 +242,6 @@ def build_correct_rate_df(
             & (df["primal_residual"] < primal_tolerances[settings])
             & (df["dual_residual"] < dual_tolerances[settings])
             & (df["duality_gap"] < gap_tolerances[settings])
-            & (df["cost_error"].abs() < cost_tolerances[settings])
             for settings in all_settings
         }
         correctness_rate_df = (