diff --git a/CHANGELOG.md b/CHANGELOG.md index 333b6c2a..40e5530b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ These are the section headers that we use: - Added support to specify a list of score values for suggestions `score` attribute. ([#98](https://github.com/argilla-io/argilla-server/pull/98)) - Added `GET /api/v1/settings` new endpoint exposing Argilla and Hugging Face settings when available. ([#127](https://github.com/argilla-io/argilla-server/pull/127)) - Added `ARGILLA_SHOW_HUGGINGFACE_SPACE_PERSISTANT_STORAGE_WARNING` new environment variable to disable warning message when Hugging Face Spaces persistent storage is disabled. ([#124](https://github.com/argilla-io/argilla-server/pull/124)) +- Added `options_order` new settings attribute to support specify an order for options in multi label selection questions. ([#133](https://github.com/argilla-io/argilla-server/pull/133)) ### Removed diff --git a/src/argilla_server/contexts/questions.py b/src/argilla_server/contexts/questions.py index e7a72e38..64c3800c 100644 --- a/src/argilla_server/contexts/questions.py +++ b/src/argilla_server/contexts/questions.py @@ -77,6 +77,7 @@ async def update_question( QuestionUpdateValidator(question_update).validate_for(question) params = question_update.dict(exclude_unset=True) + return await question.update(db, **params) diff --git a/src/argilla_server/enums.py b/src/argilla_server/enums.py index 3465c74a..13b48432 100644 --- a/src/argilla_server/enums.py +++ b/src/argilla_server/enums.py @@ -83,3 +83,8 @@ class SortOrder(str, Enum): class SimilarityOrder(str, Enum): most_similar = "most_similar" least_similar = "least_similar" + + +class OptionsOrder(str, Enum): + natural = "natural" + suggestion = "suggestion" diff --git a/src/argilla_server/schemas/v1/questions.py b/src/argilla_server/schemas/v1/questions.py index 0c95f8b4..d21379b7 100644 --- a/src/argilla_server/schemas/v1/questions.py +++ b/src/argilla_server/schemas/v1/questions.py @@ -16,6 +16,7 @@ from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID +from argilla_server.enums import OptionsOrder from argilla_server.models import QuestionType from argilla_server.pydantic_v1 import BaseModel, Field, conlist, constr, root_validator, validator from argilla_server.schemas.base import UpdateSchema @@ -197,14 +198,19 @@ class LabelSelectionSettingsUpdate(UpdateSchema): # Multi-label selection question class MultiLabelSelectionQuestionSettings(LabelSelectionQuestionSettings): type: Literal[QuestionType.multi_label_selection] + options_order: OptionsOrder = OptionsOrder.natural class MultiLabelSelectionQuestionSettingsCreate(LabelSelectionQuestionSettingsCreate): type: Literal[QuestionType.multi_label_selection] + options_order: OptionsOrder = OptionsOrder.natural class MultiLabelSelectionQuestionSettingsUpdate(LabelSelectionSettingsUpdate): type: Literal[QuestionType.multi_label_selection] + options_order: Optional[OptionsOrder] + + __non_nullable_fields__ = {"options_order"} # Ranking question diff --git a/tests/factories.py b/tests/factories.py index 9a24ca65..4eaadd14 100644 --- a/tests/factories.py +++ b/tests/factories.py @@ -16,7 +16,7 @@ import random import factory -from argilla_server.enums import FieldType, MetadataPropertyType +from argilla_server.enums import FieldType, MetadataPropertyType, OptionsOrder from argilla_server.models import ( Dataset, Field, @@ -353,6 +353,7 @@ class MultiLabelSelectionQuestionFactory(QuestionFactory): {"value": "option2", "text": "Option 2", "description": None}, {"value": "option3", "text": "Option 3", "description": None}, ], + "options_order": OptionsOrder.natural, } diff --git a/tests/unit/api/v1/datasets/questions/__init__.py b/tests/unit/api/v1/datasets/questions/__init__.py new file mode 100644 index 00000000..55be4179 --- /dev/null +++ b/tests/unit/api/v1/datasets/questions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021-present, the Recognai S.L. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/api/v1/datasets/questions/test_create_dataset_question.py b/tests/unit/api/v1/datasets/questions/test_create_dataset_question.py new file mode 100644 index 00000000..ad97e1db --- /dev/null +++ b/tests/unit/api/v1/datasets/questions/test_create_dataset_question.py @@ -0,0 +1,114 @@ +# Copyright 2021-present, the Recognai S.L. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import UUID + +import pytest +from argilla_server.enums import OptionsOrder, QuestionType +from argilla_server.models import Question +from httpx import AsyncClient +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from tests.factories import DatasetFactory + + +@pytest.mark.asyncio +class TestCreateDatasetQuestion: + def url(self, dataset_id: UUID) -> str: + return f"/api/v1/datasets/{dataset_id}/questions" + + async def test_create_dataset_multi_label_selection_question_with_options_order( + self, db: AsyncSession, async_client: AsyncClient, owner_auth_header: dict + ): + dataset = await DatasetFactory.create() + + response = await async_client.post( + self.url(dataset.id), + headers=owner_auth_header, + json={ + "name": "name", + "title": "title", + "settings": { + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + "options_order": OptionsOrder.suggestion, + }, + }, + ) + + assert response.status_code == 201 + + response_json = response.json() + assert response_json["settings"]["options_order"] == OptionsOrder.suggestion + + question = (await db.execute(select(Question).filter_by(id=UUID(response_json["id"])))).scalar_one() + assert question.settings["options_order"] == OptionsOrder.suggestion + + async def test_create_dataset_multi_label_selection_question_without_options_order( + self, db: AsyncSession, async_client: AsyncClient, owner_auth_header: dict + ): + dataset = await DatasetFactory.create() + + response = await async_client.post( + self.url(dataset.id), + headers=owner_auth_header, + json={ + "name": "name", + "title": "title", + "settings": { + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + }, + }, + ) + + assert response.status_code == 201 + + response_json = response.json() + assert response_json["settings"]["options_order"] == OptionsOrder.natural + + question = (await db.execute(select(Question).filter_by(id=UUID(response_json["id"])))).scalar_one() + assert question.settings["options_order"] == OptionsOrder.natural + + async def test_create_dataset_multi_label_selection_question_with_options_order_as_none( + self, db: AsyncSession, async_client: AsyncClient, owner_auth_header: dict + ): + dataset = await DatasetFactory.create() + + response = await async_client.post( + self.url(dataset.id), + headers=owner_auth_header, + json={ + "name": "name", + "title": "title", + "settings": { + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + "options_order": None, + }, + }, + ) + + assert response.status_code == 422 + assert (await db.execute(select(func.count(Question.id)))).scalar_one() == 0 diff --git a/tests/unit/api/v1/datasets/questions/test_list_dataset_questions.py b/tests/unit/api/v1/datasets/questions/test_list_dataset_questions.py new file mode 100644 index 00000000..75f26385 --- /dev/null +++ b/tests/unit/api/v1/datasets/questions/test_list_dataset_questions.py @@ -0,0 +1,66 @@ +# Copyright 2021-present, the Recognai S.L. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from uuid import UUID + +import pytest +from argilla_server.enums import OptionsOrder, QuestionType +from httpx import AsyncClient + +from tests.factories import QuestionFactory + + +@pytest.mark.asyncio +class TestListDatasetQuestions: + def url(self, dataset_id: UUID) -> str: + return f"/api/v1/datasets/{dataset_id}/questions" + + async def test_list_dataset_multi_label_selection_question_with_options_order( + self, async_client: AsyncClient, owner_auth_header: dict + ): + question = await QuestionFactory.create( + settings={ + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + "options_order": OptionsOrder.suggestion, + }, + ) + + response = await async_client.get(self.url(question.dataset_id), headers=owner_auth_header) + + assert response.status_code == 200 + assert response.json()["items"][0]["settings"]["options_order"] == OptionsOrder.suggestion + assert question.settings["options_order"] == OptionsOrder.suggestion + + async def test_list_dataset_multi_label_selection_question_without_options_order( + self, async_client: AsyncClient, owner_auth_header: dict + ): + question = await QuestionFactory.create( + settings={ + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + }, + ) + + response = await async_client.get(self.url(question.dataset_id), headers=owner_auth_header) + + assert response.status_code == 200 + assert response.json()["items"][0]["settings"]["options_order"] == OptionsOrder.natural + assert "options_order" not in question.settings diff --git a/tests/unit/api/v1/questions/__init__.py b/tests/unit/api/v1/questions/__init__.py new file mode 100644 index 00000000..55be4179 --- /dev/null +++ b/tests/unit/api/v1/questions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021-present, the Recognai S.L. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/unit/api/v1/questions/test_update_question.py b/tests/unit/api/v1/questions/test_update_question.py index 9a2878e9..94981128 100644 --- a/tests/unit/api/v1/questions/test_update_question.py +++ b/tests/unit/api/v1/questions/test_update_question.py @@ -15,10 +15,13 @@ from uuid import UUID import pytest -from argilla_server.enums import QuestionType +from argilla_server.enums import OptionsOrder, QuestionType +from argilla_server.models import Question from httpx import AsyncClient +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession -from tests.factories import LabelSelectionQuestionFactory, SpanQuestionFactory, TextQuestionFactory +from tests.factories import LabelSelectionQuestionFactory, QuestionFactory, SpanQuestionFactory, TextQuestionFactory @pytest.mark.asyncio @@ -91,6 +94,87 @@ async def test_update_question_with_different_options(self, async_client: AsyncC "detail": "the option values cannot be modified. found unexpected option values: ['label-a', 'label-b', 'label-c']" } + async def test_update_multi_label_selection_question_with_options_order( + self, db: AsyncSession, async_client: AsyncClient, owner_auth_header: dict + ): + question = await QuestionFactory.create( + settings={ + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + "options_order": OptionsOrder.natural, + } + ) + + response = await async_client.patch( + self.url(question.id), + headers=owner_auth_header, + json={ + "settings": { + "type": QuestionType.multi_label_selection, + "options_order": OptionsOrder.suggestion, + }, + }, + ) + + assert response.status_code == 200 + assert response.json()["settings"]["options_order"] == OptionsOrder.suggestion + assert question.settings["options_order"] == OptionsOrder.suggestion + + async def test_update_multi_label_selection_question_without_options_order( + self, async_client: AsyncClient, owner_auth_header: dict + ): + question = await QuestionFactory.create( + settings={ + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + "options_order": OptionsOrder.suggestion, + } + ) + + response = await async_client.patch( + self.url(question.id), + headers=owner_auth_header, + json={"type": QuestionType.multi_label_selection}, + ) + + assert response.status_code == 200 + assert response.json()["settings"]["options_order"] == OptionsOrder.suggestion + assert question.settings["options_order"] == OptionsOrder.suggestion + + async def test_update_multi_label_selection_question_with_options_order_as_none( + self, async_client: AsyncClient, owner_auth_header: dict + ): + question = await QuestionFactory.create( + settings={ + "type": QuestionType.multi_label_selection, + "options": [ + {"value": "label-a", "text": "Label A"}, + {"value": "label-b", "text": "Label B"}, + ], + "options_order": OptionsOrder.natural, + } + ) + + response = await async_client.patch( + self.url(question.id), + headers=owner_auth_header, + json={ + "settings": { + "type": QuestionType.multi_label_selection, + "options_order": None, + }, + }, + ) + + assert response.status_code == 422 + assert question.settings["options_order"] == OptionsOrder.natural + async def test_update_question_with_more_visible_options_than_allowed( self, async_client: AsyncClient, owner_auth_header: dict ): diff --git a/tests/unit/api/v1/test_datasets.py b/tests/unit/api/v1/test_datasets.py index 6fd60eff..cfcdabf5 100644 --- a/tests/unit/api/v1/test_datasets.py +++ b/tests/unit/api/v1/test_datasets.py @@ -23,6 +23,7 @@ from argilla_server.constants import API_KEY_HEADER_NAME from argilla_server.enums import ( DatasetStatus, + OptionsOrder, RecordInclude, ResponseStatusFilter, SimilarityOrder, @@ -368,6 +369,7 @@ async def test_list_dataset_questions(self, async_client: "AsyncClient", owner_a {"value": "b", "text": "b", "description": "b"}, ], "visible_options": None, + "options_order": OptionsOrder.natural, }, ), ], diff --git a/tests/unit/api/v1/test_questions.py b/tests/unit/api/v1/test_questions.py index 920b65cf..4c1e53ca 100644 --- a/tests/unit/api/v1/test_questions.py +++ b/tests/unit/api/v1/test_questions.py @@ -17,6 +17,7 @@ import pytest from argilla_server.constants import API_KEY_HEADER_NAME +from argilla_server.enums import OptionsOrder from argilla_server.models import DatasetStatus, Question, UserRole from argilla_server.schemas.v1.questions import QUESTION_CREATE_DESCRIPTION_MAX_LENGTH, QUESTION_CREATE_TITLE_MAX_LENGTH from sqlalchemy import func, select @@ -96,7 +97,12 @@ ), ( MultiLabelSelectionQuestionFactory, - {"settings": {"type": "multi_label_selection", "visible_options": 3}}, + { + "settings": { + "type": "multi_label_selection", + "visible_options": 3, + }, + }, { "type": "multi_label_selection", "options": [ @@ -105,6 +111,7 @@ {"value": "option3", "text": "Option 3", "description": None}, ], "visible_options": 3, + "options_order": OptionsOrder.natural, }, ), ( @@ -112,22 +119,23 @@ { "settings": { "type": "multi_label_selection", - "visible_options": None, "options": [ {"value": "option3", "text": "Option 3", "description": None}, {"value": "option1", "text": "Option 1", "description": None}, {"value": "option2", "text": "Option 2", "description": None}, ], + "visible_options": None, } }, { "type": "multi_label_selection", - "visible_options": None, "options": [ {"value": "option3", "text": "Option 3", "description": None}, {"value": "option1", "text": "Option 1", "description": None}, {"value": "option2", "text": "Option 2", "description": None}, ], + "visible_options": None, + "options_order": OptionsOrder.natural, }, ), (