diff --git a/evals/elsuite/schelling_point/eval.py b/evals/elsuite/schelling_point/eval.py index 46d5371af1..78f6be68c6 100644 --- a/evals/elsuite/schelling_point/eval.py +++ b/evals/elsuite/schelling_point/eval.py @@ -9,6 +9,7 @@ from evals.elsuite.schelling_point.prompts import sys_prompts_ci, sys_prompts_no_ci from evals.elsuite.schelling_point.utils import get_response from evals.eval import Eval +from evals.solvers.solver import Solver class SchellingPoint(Eval): @@ -22,6 +23,9 @@ def __init__( *args, **kwargs, ): + if any([isinstance(completion_fn, Solver) for completion_fn in completion_fns]): + raise NotImplementedError("SchellingPoint does not currently support Solvers") + super().__init__(completion_fns, *args, **kwargs) random.seed(seed) @@ -34,7 +38,6 @@ def __init__( self.temperature = temperature - self.completion_fns = completion_fns if len(self.completion_fns) == 1: self.completion_fns = self.completion_fns * n_copies assert len(self.completion_fns) == n_copies, "Must provide n_copies completion_fns" diff --git a/evals/registry/data/schelling_point/mix.jsonl b/evals/registry/data/schelling_point/mix.jsonl index f1f60c2c42..d7b28c1d83 100644 --- a/evals/registry/data/schelling_point/mix.jsonl +++ b/evals/registry/data/schelling_point/mix.jsonl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5593a9f39fe34336b91008f7f7fddf9ddd56e3fda45f973d125ff2bfd0ac5cf7 -size 16129977 +oid sha256:a7d2c46729473fbd01dca750912d334487a6809c4dc8cfa58e8c7dc34f75fc1f +size 16074461 diff --git a/evals/registry/data/schelling_point/owt_5.jsonl b/evals/registry/data/schelling_point/owt_5.jsonl index 0a3f40c2db..6fa67ccf70 100644 --- a/evals/registry/data/schelling_point/owt_5.jsonl +++ b/evals/registry/data/schelling_point/owt_5.jsonl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:095ceeb4ac1248bcaac4952056c6ccd5872410fc6a8b55d46244d57f27cfbb8b -size 7620781 +oid sha256:929794988b95c964c39fe2aad0ff043908d904ce4a92a79ac23647a7e187fe4e +size 7565265