feat: add multi label selection question to the API

CHANGELOG.md

@@ -23,6 +23,8 @@ These are the section headers that we use:
 
 - Added boolean `use_markdown` property to `TextFieldSettings` model.
 - Added boolean `use_markdown` property to `TextQuestionSettings` model.
+- Added `LabelSelectionQuestionSettings` class allowing to create label selection (single-choice) questions in the API.
+- Added `MultiLabelSelectionQuestionSettings` class allowing to create multi-label selection (multi-choice) questions in the API.
 
 ## [1.8.0-dev](https://github.com/argilla-io/argilla/compare/v1.7.0...v1.8.0)
 

src/argilla/server/models/questions.py

@@ -27,6 +27,7 @@ class QuestionType(str, Enum):
     text = "text"
     rating = "rating"
     label_selection = "label_selection"
+    multi_label_selection = "multi_label_selection"
 
 
 class ResponseValue(BaseModel):
@@ -81,7 +82,23 @@ class LabelSelectionQuestionSettings(ValidOptionCheckerMixin[str], BaseQuestionS
     visible_options: Optional[int] = None
 
 
+class MultiLabelSelectionQuestionSettings(LabelSelectionQuestionSettings):
+    type: Literal[QuestionType.multi_label_selection]
+
+    def check_response(self, response: ResponseValue):
+        if not isinstance(response.value, list):
+            raise ValueError(f"Expected list of values, found {type(response.value)}")
+        invalid_options = sorted(list(set(response.value) - set(self.option_values)))
+        if invalid_options:
+            raise ValueError(f"{invalid_options!r} are not valid options.\nValid options are: {self.option_values!r}")
+
+
 QuestionSettings = Annotated[
-    Union[TextQuestionSettings, RatingQuestionSettings, LabelSelectionQuestionSettings],
+    Union[
+        TextQuestionSettings,
+        RatingQuestionSettings,
+        LabelSelectionQuestionSettings,
+        MultiLabelSelectionQuestionSettings,
+    ],
     Field(..., discriminator="type"),
 ]

src/argilla/server/schemas/v1/datasets.py

@@ -185,8 +185,17 @@ class LabelSelectionQuestionSettings(BaseModel):
     visible_options: Optional[PositiveInt] = None
 
 
+class MultiLabelSelectionQuestionSettings(LabelSelectionQuestionSettings):
+    type: Literal[QuestionType.multi_label_selection]
+
+
 QuestionSettings = Annotated[
-    Union[TextQuestionSettings, RatingQuestionSettings, LabelSelectionQuestionSettings],
+    Union[
+        TextQuestionSettings,
+        RatingQuestionSettings,
+        LabelSelectionQuestionSettings,
+        MultiLabelSelectionQuestionSettings,
+    ],
     PydanticField(discriminator="type"),
 ]
 

src/argilla/server/search_engine.py

@@ -102,7 +102,7 @@ def _field_mapping_for_question(self, question: Question):
         if settings.type == QuestionType.rating:
             # See https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html
             return {"type": "integer"}
-        elif settings.type in [QuestionType.text, QuestionType.label_selection]:
+        elif settings.type in [QuestionType.text, QuestionType.label_selection, QuestionType.multi_label_selection]:
             # TODO: Review mapping for label selection. Could make sense to use `keyword` mapping instead. See https://www.elastic.co/guide/en/elasticsearch/reference/current/keyword.html
             # See https://www.elastic.co/guide/en/elasticsearch/reference/current/text.html
             return {"type": "text", "index": False}

tests/factories.py

@@ -159,3 +159,14 @@ class LabelSelectionQuestionFactory(QuestionFactory):
             {"value": "option3", "text": "Option 3"},
         ],
     }
+
+
+class MultiLabelSelectionQuestionFactory(QuestionFactory):
+    settings = {
+        "type": QuestionType.multi_label_selection.value,
+        "options": [
+            {"value": "option1", "text": "Option 1"},
+            {"value": "option2", "text": "Option 2"},
+            {"value": "option3", "text": "Option 3"},
+        ],
+    }

tests/server/api/v1/test_datasets.py

@@ -35,11 +35,8 @@
     FIELD_CREATE_NAME_MAX_LENGTH,
     FIELD_CREATE_TITLE_MAX_LENGTH,
     LABEL_SELECTION_DESCRIPTION_MAX_LENGTH,
-    LABEL_SELECTION_DESCRIPTION_MIN_LENGTH,
     LABEL_SELECTION_TEXT_MAX_LENGTH,
-    LABEL_SELECTION_TEXT_MIN_LENGTH,
     LABEL_SELECTION_VALUE_MAX_LENGHT,
-    LABEL_SELECTION_VALUE_MIN_LENGHT,
     QUESTION_CREATE_DESCRIPTION_MAX_LENGTH,
     QUESTION_CREATE_NAME_MAX_LENGTH,
     QUESTION_CREATE_TITLE_MAX_LENGTH,

tests/server/api/v1/test_records.py

@@ -26,6 +26,7 @@
     AnnotatorFactory,
     DatasetFactory,
     LabelSelectionQuestionFactory,
+    MultiLabelSelectionQuestionFactory,
     RatingQuestionFactory,
     RecordFactory,
     ResponseFactory,
@@ -43,11 +44,15 @@ def create_text_questions(dataset: "Dataset") -> None:
 
 
 def create_rating_questions(dataset: "Dataset") -> None:
-    RatingQuestionFactory.create(name="rating_question", dataset=dataset, required=False)
+    RatingQuestionFactory.create(name="rating_question", dataset=dataset)
 
 
 def create_label_selection_questions(dataset: "Dataset") -> None:
-    LabelSelectionQuestionFactory.create(name="label_selection_question", dataset=dataset, required=False)
+    LabelSelectionQuestionFactory.create(name="label_selection_question", dataset=dataset)
+
+
+def create_multi_label_selection_questions(dataset: "Dataset") -> None:
+    MultiLabelSelectionQuestionFactory.create(name="multi_label_selection_question", dataset=dataset)
 
 
 @pytest.mark.parametrize(
@@ -78,6 +83,22 @@ def create_label_selection_questions(dataset: "Dataset") -> None:
                 },
             },
         ),
+        (
+            create_multi_label_selection_questions,
+            {
+                "values": {
+                    "multi_label_selection_question": {"value": ["option1"]},
+                },
+            },
+        ),
+        (
+            create_multi_label_selection_questions,
+            {
+                "values": {
+                    "multi_label_selection_question": {"value": ["option1", "option2"]},
+                },
+            },
+        ),
     ],
 )
 def test_create_record_response(
@@ -109,50 +130,18 @@ def test_create_record_response(
     }
 
 
-@pytest.mark.parametrize(
-    "create_questions_func, responses",
-    [
-        (
-            create_text_questions,
-            {
-                "values": {
-                    "input_ok": {"value": "yes"},
-                    "unknown_question": {"value": "Test"},
-                },
-            },
-        ),
-        (
-            create_rating_questions,
-            {
-                "values": {
-                    "rating_question": {"value": 5},
-                    "unknown_question": {"value": "Test"},
-                },
-            },
-        ),
-        (
-            create_label_selection_questions,
-            {
-                "values": {
-                    "label_selection_question": {"value": "option1"},
-                    "unknown_question": {"value": "Test"},
-                },
-            },
-        ),
-    ],
-)
-def test_create_record_response_with_extra_question_responses(
-    client: TestClient,
-    db: Session,
-    admin_auth_header: dict,
-    create_questions_func: Callable[["Dataset"], None],
-    responses: dict,
-):
+def test_create_record_response_with_extra_question_responses(client: TestClient, db: Session, admin_auth_header: dict):
     dataset = DatasetFactory.create()
-    create_questions_func(dataset)
+    create_text_questions(dataset)
     record = RecordFactory.create(dataset=dataset)
 
-    response_json = {**responses, "status": "submitted"}
+    response_json = {
+        "values": {
+            "input_ok": {"value": "yes"},
+            "unknown_question": {"value": "Test"},
+        },
+        "status": "submitted",
+    }
     response = client.post(f"/api/v1/records/{record.id}/responses", headers=admin_auth_header, json=response_json)
 
     assert response.status_code == 422
@@ -190,6 +179,24 @@ def test_create_record_response_with_extra_question_responses(
             },
             "False is not a valid option.\nValid options are: ['option1', 'option2', 'option3']",
         ),
+        (
+            create_multi_label_selection_questions,
+            {
+                "values": {
+                    "multi_label_selection_question": {"value": "wrong-type"},
+                },
+            },
+            "Expected list of values, found <class 'str'>",
+        ),
+        (
+            create_multi_label_selection_questions,
+            {
+                "values": {
+                    "multi_label_selection_question": {"value": ["option4", "option5"]},
+                },
+            },
+            "['option4', 'option5'] are not valid options.\nValid options are: ['option1', 'option2', 'option3']",
+        ),
     ],
 )
 def test_create_record_response_with_wrong_response_value(