diff --git a/.env b/.env index 7df764c..b8e5f6d 100644 --- a/.env +++ b/.env @@ -3,6 +3,7 @@ COMPOSE_PATH_SEPARATOR=";" COMPOSE_FILE="docker-compose.yml;docker/dev.yml" +RESTART_POLICY=no # port expose for the API API_EXPOSE="127.0.0.1:80" diff --git a/.github/labeler.yml b/.github/labeler.yml index b3849e4..7b18fe2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,27 +1,33 @@ github: -- .github/**/* - -documentation: -- any: ['*.MD'] +- changed-files: + - any-glob-to-any-file: '.github/**/*' translations: -- i18n/**/* -- app/i18n.py +- changed-files: + - any-glob-to-any-file: 'i18n/**/*' + - any-glob-to-any-file: 'app/i18n.py' tests: -- tests/**/* +- changed-files: + - any-glob-to-any-file: 'tests/**/*' docker: -- docker/**/* -- Dockerfile -- docker-compose.yml +- changed-files: + - any-glob-to-any-file: 'docker/**/*' + - any-glob-to-any-file: 'Dockerfile' + - any-glob-to-any-file: 'docker-compose.yml' api: -- app/knowledge_panels.py -- app/main.py -- app/off.py -- app/wikidata_utils.py -- app/settings.py +- changed-files: + - any-glob-to-any-file: 'app/knowledge_panels.py' + - any-glob-to-any-file: 'app/main.py' + - any-glob-to-any-file: 'app/off.py' + - any-glob-to-any-file: 'app/wikidata_utils.py' + - any-glob-to-any-file: 'app/settings.py' models: -- app/models.py +- changed-files: + - any-glob-to-any-file: 'app/models.py' + +documentation: + - any-glob-to-any-file: ['*.MD'] diff --git a/.github/workflows/container-deploy.yml b/.github/workflows/container-deploy.yml index 3a9449c..376d845 100644 --- a/.github/workflows/container-deploy.yml +++ b/.github/workflows/container-deploy.yml @@ -119,6 +119,7 @@ jobs: echo "COMPOSE_HTTP_TIMEOUT=120" >> .env # echo "COMPOSE_PROJECT_NAME=${{ matrix.env }}" >> .env echo "COMPOSE_PATH_SEPARATOR=;" >> .env + echo "RESTART_POLICY=always" >> .env echo "COMPOSE_FILE=docker-compose.yml;docker/prod.yml" >> .env echo "TAG=sha-${{ github.sha }}" >> .env @@ -201,7 +202,7 @@ jobs: cd ${{ matrix.env }} docker system prune -af - - uses: frankie567/grafana-annotation-action@v1.0.2 + - uses: basos9/grafana-annotation-action@v1.0.3 if: ${{ always() }} with: apiHost: https://grafana.openfoodfacts.org diff --git a/app/i18n.py b/app/i18n.py index 7bd81da..88a5e30 100644 --- a/app/i18n.py +++ b/app/i18n.py @@ -1,5 +1,6 @@ """i18n handling """ + import contextlib import contextvars import gettext diff --git a/docker-compose.yml b/docker-compose.yml index d4651cc..7807642 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ version: '3' services: facets-api: image: ghcr.io/openfoodfacts/facets-knowledge-panels:${TAG:-dev} + # default to restart, use RESTART_POLICY=no in dev + restart: ${RESTART_POLICY:-always} ports: - "${API_EXPOSE}:80" environment: diff --git a/requirements.txt b/requirements.txt index 10910e3..3a60271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -aiohttp==3.8.5 +aiohttp==3.9.4 async-lru==2.0.4 # async lru_cache https://github.com/aio-libs/async-lru asyncer==0.0.1 -fastapi==0.101.0 -jinja2==3.1.2 +fastapi==0.109.1 +jinja2==3.1.4 fastapi_utils==0.2.1 inflect==7.0.0 pycountry==22.3.5 @@ -15,7 +15,7 @@ openfoodfacts==0.1.5 prometheus-fastapi-instrumentator==6.1.0 # dev -black==22.6.0 +black==24.3.0 pytest==7.4.0 pytest-mock==3.11.1 pytest-asyncio==0.21.1 diff --git a/tests/test_dataquality_kp.py b/tests/test_dataquality_kp.py new file mode 100644 index 0000000..cb1f34b --- /dev/null +++ b/tests/test_dataquality_kp.py @@ -0,0 +1,335 @@ +"""seperate file for tests of dataquality_kp""" + +import aiohttp +import pytest + +from app.i18n import active_translation +from app.knowledge_panels import KnowledgePanels + +from .test_utils import mock_async_get_factory, tidy_html + + +@pytest.fixture(autouse=True) +def auto_activate_lang(): + """auto activate translations for each function""" + with active_translation(): + yield + + +async def test_data_quality_kp_with_country(monkeypatch): + """test_data_quality_kp_with_country""" + expected_url = "https://tr-en.openfoodfacts.org/data-quality-errors.json" + base_url = "https://tr-en.openfoodfacts.org/data-quality-errors" + json_content = { + "count": 129, + "tags": [ + { + "id": "en:ecoscore-production-system-no-label", + "known": 0, + "name": "ecoscore-production-system-no-label", + "products": 1848, + "url": f"{base_url}/ecoscore-production-system-no-label", + }, + { + "id": "en:no-packaging-data", + "known": 0, + "name": "no-packaging-data", + "products": 1788, + "url": f"{base_url}/no-packaging-data", + }, + { + "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "known": 0, + "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "products": 1778, + "url": ( + f"{base_url}/" "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown" + ), + }, + ], + } + + monkeypatch.setattr( + aiohttp.ClientSession, + "get", + mock_async_get_factory(expected_url, json_content=json_content), + ) + result = await KnowledgePanels( + facet="country", value="Turkey", country="Hungary" + ).data_quality_kp() + first_element = result["Quality"]["elements"][0] + first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) + expected_text = """ + + """ # noqa: E501 # allow long lines + # assert html separately to have better output in case of error + assert first_element["text_element"]["html"] == tidy_html(expected_text) + # now replace it for concision of output + first_element["text_element"]["html"] = "ok" + assert result == { + "Quality": { + "elements": [ + { + "element_type": "text", + "text_element": { + "html": "ok", + "source_text": "Data-quality issues", + "source_url": "https://tr-en.openfoodfacts.org/data-quality-errors", + }, + } + ], + "title_element": {"title": "Data-quality issues related to Turkey "}, + } + } + + +async def test_data_quality_kp_with_one_facet_and_value(monkeypatch): + """test_data_quality_kp_with_one_facet_and_value""" + expected_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors.json" + base_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors" + json_content = { + "count": 181, + "tags": [ + { + "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "known": 0, + "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "products": 7898, + "url": ( + f"{base_url}/" "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown" + ), + }, + { + "id": "en:ecoscore-production-system-no-label", + "known": 0, + "name": "ecoscore-production-system-no-label", + "products": 7883, + "url": f"{base_url}/ecoscore-production-system-no-label", + }, + { + "id": "en:no-packaging-data", + "known": 0, + "name": "no-packaging-data", + "products": 6406, + "url": f"{base_url}/no-packaging-data", + }, + ], + } + + monkeypatch.setattr( + aiohttp.ClientSession, + "get", + mock_async_get_factory(expected_url, json_content=json_content), + ) + result = await KnowledgePanels(facet="brand", value="lidl").data_quality_kp() + first_element = result["Quality"]["elements"][0] + first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) + expected_text = """ + + """ # noqa: E501 # allow long lines + # assert html separately to have better output in case of error + assert first_element["text_element"]["html"] == tidy_html(expected_text) + # now replace it for concision of output + first_element["text_element"]["html"] = "ok" + assert result == { + "Quality": { + "elements": [ + { + "element_type": "text", + "text_element": { + "html": "ok", + "source_text": "Data-quality issues", + "source_url": "https://world.openfoodfacts.org/brand/lidl/" + + "data-quality-errors", + }, + } + ], + "title_element": {"title": "Data-quality issues related to brand lidl"}, + } + } + + +async def test_data_quality_kp_with_one_facet_and_value_plural_facet(monkeypatch): + """test_data_quality_kp_with_one_facet_and_value_plural_facet""" + expected_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors.json" + base_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors" + json_content = { + "count": 181, + "tags": [ + { + "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "known": 0, + "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "products": 7898, + "url": ( + f"{base_url}/" "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown" + ), + }, + { + "id": "en:ecoscore-production-system-no-label", + "known": 0, + "name": "ecoscore-production-system-no-label", + "products": 7883, + "url": f"{base_url}/ecoscore-production-system-no-label", + }, + { + "id": "en:no-packaging-data", + "known": 0, + "name": "no-packaging-data", + "products": 6406, + "url": f"{base_url}/no-packaging-data", + }, + ], + } + + monkeypatch.setattr( + aiohttp.ClientSession, + "get", + mock_async_get_factory(expected_url, json_content=json_content), + ) + result = await KnowledgePanels(facet="brands", value="lidl").data_quality_kp() + first_element = result["Quality"]["elements"][0] + first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) + expected_text = """ + + """ # noqa: E501 # allow long lines + # assert html separately to have better output in case of error + assert first_element["text_element"]["html"] == tidy_html(expected_text) + # now replace it for concision of output + first_element["text_element"]["html"] = "ok" + assert result == { + "Quality": { + "elements": [ + { + "element_type": "text", + "text_element": { + "html": "ok", + "source_text": "Data-quality issues", + "source_url": "https://world.openfoodfacts.org/brand/lidl/" + + "data-quality-errors", + }, + } + ], + "title_element": {"title": "Data-quality issues related to brand lidl"}, + } + } + + +async def test_data_quality_kp_with_all_tags(monkeypatch): + """test_data_quality_kp_with_all_tags""" + expected_url = ( + "https://world.openfoodfacts.org/category/beers/brand/budweiser/data-quality-errors.json" + ) + json_content = { + "count": 24, + "tags": [ + { + "id": "en:alcoholic-beverages-category-without-alcohol-value", + "known": 0, + "name": "alcoholic-beverages-category-without-alcohol-value", + "products": 13, + "url": "https://world.openfoodfacts.org/category/beers/" + + "data-quality-errors/alcoholic-beverages-category-without-alcohol-value", + # noqa: E501 # allow long lines + }, + { + "id": "en:ecoscore-production-system-no-label", + "known": 0, + "name": "ecoscore-production-system-no-label", + "products": 13, + "url": "https://world.openfoodfacts.org/category/beers/" + "data-quality-errors/ecoscore-production-system-no-label", + # noqa: E501 # allow long lines + }, + { + "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "known": 0, + "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + "products": 12, + "url": "https://world.openfoodfacts.org/category/beers/data-quality-errors/" + "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", + # noqa: E501 # allow long lines + }, + ], + } + + monkeypatch.setattr( + aiohttp.ClientSession, + "get", + mock_async_get_factory(expected_url, json_content=json_content), + ) + result = await KnowledgePanels( + facet="category", value="beers", sec_facet="brand", sec_value="budweiser" + ).data_quality_kp() + first_element = result["Quality"]["elements"][0] + first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) + expected_text = """ + + """ # noqa: E501 # allow long lines + # assert html separately to have better output in case of error + assert first_element["text_element"]["html"] == tidy_html(expected_text) + # now replace it for concision of output + first_element["text_element"]["html"] = "ok" + assert result == { + "Quality": { + "elements": [ + { + "element_type": "text", + "text_element": { + "html": "ok", + "source_text": "Data-quality issues", + "source_url": "https://world.openfoodfacts.org/category/beers/" + + "brand/budweiser/data-quality-errors", + # noqa: E501 + }, + } + ], + "title_element": { + "title": "Data-quality issues related to category beers brand budweiser" + }, + } + } diff --git a/tests/test_hunger_game_kp.py b/tests/test_hunger_game_kp.py new file mode 100644 index 0000000..f8b441f --- /dev/null +++ b/tests/test_hunger_game_kp.py @@ -0,0 +1,284 @@ +"""separate file for tests of hungergame_kp""" + +import pytest + +from app.i18n import active_translation +from app.knowledge_panels import KnowledgePanels + + +@pytest.fixture(autouse=True) +def auto_activate_lang(): + """auto activate translations for each function""" + with active_translation(): + yield + + +async def test_hunger_game_kp_no_result(): + """not all facets are compatible with hunger game""" + result = await KnowledgePanels( + facet="allergen", + value="gluten", + sec_facet="mineral", + sec_value="zinc", + ).hunger_game_kp() + assert result is None + + +async def test_hunger_game_kp_with_filter_value_and_country(): + """test_hunger_game_kp_with_filter_value_and_country""" + html = ( + "" + ) + result = await KnowledgePanels( + facet="country", value="germany", country="france" + ).hunger_game_kp() + assert result == { + "HungerGames": { + "elements": [{"element_type": "text", "text_element": {"html": html}}], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_with_category(): + """test_hunger_game_kp_with_category""" + html = ( + "" + ) + result = await KnowledgePanels(facet="category").hunger_game_kp() + assert result == { + "HungerGames": { + "elements": [{"element_type": "text", "text_element": {"html": html}}], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_category_with_country(): + """test_hunger_game_kp_category_with_country""" + html0 = ( + "" + ) + html1 = ( + "" + ) + result = await KnowledgePanels(facet="category", country="france").hunger_game_kp() + assert result == { + "HungerGames": { + "elements": [ + {"element_type": "text", "text_element": {"html": html0}}, + {"element_type": "text", "text_element": {"html": html1}}, + ], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_category_with_value(): + """test_hunger_game_kp_category_with_value""" + html = ( + "" + ) + result = await KnowledgePanels(facet="category", value="en:beers").hunger_game_kp() + assert result == { + "HungerGames": { + "elements": [{"element_type": "text", "text_element": {"html": html}}], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_brand_with_value(): + """test_hunger_game_kp_brand_with_value""" + html = ( + "" + ) + result = await KnowledgePanels(facet="brand", value="nestle").hunger_game_kp() + assert result == { + "HungerGames": { + "elements": [{"element_type": "text", "text_element": {"html": html}}], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_label_with_value(): + """test_hunger_game_kp_label_with_value""" + html = ( + "" + ) + result = await KnowledgePanels(facet="label", value="en:organic").hunger_game_kp() + assert result == { + "HungerGames": { + "elements": [{"element_type": "text", "text_element": {"html": html}}], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_label_with_value_plural_facet(): + """test_hunger_game_kp_label_with_value_plural_facet""" + html = ( + "" + ) + result = await KnowledgePanels(facet="labels", value="en:organic").hunger_game_kp() + assert result == { + "HungerGames": { + "elements": [{"element_type": "text", "text_element": {"html": html}}], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_HungerGame_double_country_and_value(): + """test_HungerGame_double_country_and_value""" + # facet country have priority + html1 = ( + "" + ) + html2 = ( + "" + ) + kp = KnowledgePanels( + facet="country", + value="en:france", + country="germany", + sec_facet="category", + sec_value="beers", + ) + result = await kp.hunger_game_kp() + assert result == { + "HungerGames": { + "title_element": {"title": "Hunger games"}, + "elements": [ + { + "element_type": "text", + "text_element": { + "html": html1, + }, + }, + { + "element_type": "text", + "text_element": { + "html": html2, + }, + }, + ], + } + } + + +async def test_hunger_game_kp_with_all_tag_1(): + """test_hunger_game_kp_with_all_tag_1""" + html0 = ( + "" + ) + html1 = ( + "" # noqa: E501 + ) + assert await KnowledgePanels( + facet="category", + value="en:beers", + sec_facet="brand", + sec_value="lidl", + country="france", + ).hunger_game_kp() == { + "HungerGames": { + "elements": [ + {"element_type": "text", "text_element": {"html": html0}}, + {"element_type": "text", "text_element": {"html": html1}}, + ], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_with_all_tag_2(): + """test_hunger_game_kp_with_all_tag_2""" + html0 = ( + "" + ) + html1 = ( + "" + ) + assert await KnowledgePanels( + facet="brand", + sec_facet="category", + sec_value="en:coffees", + ).hunger_game_kp() == { + "HungerGames": { + "elements": [ + {"element_type": "text", "text_element": {"html": html0}}, + {"element_type": "text", "text_element": {"html": html1}}, + ], + "title_element": {"title": "Hunger games"}, + } + } + + +async def test_hunger_game_kp_with_all_tag_3(): + """test_hunger_game_kp_with_all_tag_3""" + html0 = ( + "" + ) + html1 = ( + "" + ) + html2 = ( + "" + ) + assert await KnowledgePanels( + facet="category", + value="en:meals", + sec_facet="label", + sec_value="vegan", + country="italy", + ).hunger_game_kp() == { + "HungerGames": { + "elements": [ + {"element_type": "text", "text_element": {"html": html0}}, + {"element_type": "text", "text_element": {"html": html1}}, + {"element_type": "text", "text_element": {"html": html2}}, + ], + "title_element": {"title": "Hunger games"}, + } + } diff --git a/tests/test_knowledge_panels.py b/tests/test_knowledge_panels.py deleted file mode 100644 index ab10556..0000000 --- a/tests/test_knowledge_panels.py +++ /dev/null @@ -1,1039 +0,0 @@ -import aiohttp -import pytest -import wikidata.client - -from app.i18n import active_translation -from app.knowledge_panels import KnowledgePanels -from app.wikidata_utils import wikidata_props - -from .test_utils import DictAttr, mock_async_get_factory, mock_wikidata_get, tidy_html - - -@pytest.fixture(autouse=True) -def auto_activate_lang(): - """auto activate translations for each function""" - with active_translation(): - yield - - -async def test_hunger_game_kp_no_result(): - # not all facets are compatible with hunger game - result = await KnowledgePanels( - facet="allergen", - value="gluten", - sec_facet="mineral", - sec_value="zinc", - ).hunger_game_kp() - assert result is None - - -async def test_hunger_game_kp_with_filter_value_and_country(): - html = ( - "" - ) - result = await KnowledgePanels( - facet="country", value="germany", country="france" - ).hunger_game_kp() - assert result == { - "HungerGames": { - "elements": [{"element_type": "text", "text_element": {"html": html}}], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_with_category(): - html = ( - "" - ) - result = await KnowledgePanels(facet="category").hunger_game_kp() - assert result == { - "HungerGames": { - "elements": [{"element_type": "text", "text_element": {"html": html}}], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_category_with_country(): - html0 = ( - "" - ) - html1 = ( - "" - ) - result = await KnowledgePanels(facet="category", country="france").hunger_game_kp() - assert result == { - "HungerGames": { - "elements": [ - {"element_type": "text", "text_element": {"html": html0}}, - {"element_type": "text", "text_element": {"html": html1}}, - ], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_category_with_value(): - html = ( - "" - ) - result = await KnowledgePanels(facet="category", value="en:beers").hunger_game_kp() - assert result == { - "HungerGames": { - "elements": [{"element_type": "text", "text_element": {"html": html}}], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_brand_with_value(): - html = ( - "" - ) - result = await KnowledgePanels(facet="brand", value="nestle").hunger_game_kp() - assert result == { - "HungerGames": { - "elements": [{"element_type": "text", "text_element": {"html": html}}], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_label_with_value(): - html = ( - "" - ) - result = await KnowledgePanels(facet="label", value="en:organic").hunger_game_kp() - assert result == { - "HungerGames": { - "elements": [{"element_type": "text", "text_element": {"html": html}}], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_label_with_value_plural_facet(): - html = ( - "" - ) - result = await KnowledgePanels(facet="labels", value="en:organic").hunger_game_kp() - assert result == { - "HungerGames": { - "elements": [{"element_type": "text", "text_element": {"html": html}}], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_HungerGame_double_country_and_value(): - # facet country have priority - html1 = ( - "" - ) - html2 = ( - "" - ) - kp = KnowledgePanels( - facet="country", - value="en:france", - country="germany", - sec_facet="category", - sec_value="beers", - ) - result = await kp.hunger_game_kp() - assert result == { - "HungerGames": { - "title_element": {"title": "Hunger games"}, - "elements": [ - { - "element_type": "text", - "text_element": { - "html": html1, - }, - }, - { - "element_type": "text", - "text_element": { - "html": html2, - }, - }, - ], - } - } - - -async def test_hunger_game_kp_with_all_tag_1(): - html0 = ( - "" - ) - html1 = ( - "" # noqa: E501 - ) - assert await KnowledgePanels( - facet="category", - value="en:beers", - sec_facet="brand", - sec_value="lidl", - country="france", - ).hunger_game_kp() == { - "HungerGames": { - "elements": [ - {"element_type": "text", "text_element": {"html": html0}}, - {"element_type": "text", "text_element": {"html": html1}}, - ], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_with_all_tag_2(): - - html0 = ( - "" - ) - html1 = ( - "" - ) - assert await KnowledgePanels( - facet="brand", - sec_facet="category", - sec_value="en:coffees", - ).hunger_game_kp() == { - "HungerGames": { - "elements": [ - {"element_type": "text", "text_element": {"html": html0}}, - {"element_type": "text", "text_element": {"html": html1}}, - ], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_hunger_game_kp_with_all_tag_3(): - html0 = ( - "" - ) - html1 = ( - "" - ) - html2 = ( - "" - ) - assert await KnowledgePanels( - facet="category", - value="en:meals", - sec_facet="label", - sec_value="vegan", - country="italy", - ).hunger_game_kp() == { - "HungerGames": { - "elements": [ - {"element_type": "text", "text_element": {"html": html0}}, - {"element_type": "text", "text_element": {"html": html1}}, - {"element_type": "text", "text_element": {"html": html2}}, - ], - "title_element": {"title": "Hunger games"}, - } - } - - -async def test_data_quality_kp_with_country(monkeypatch): - expected_url = "https://tr-en.openfoodfacts.org/data-quality-errors.json" - base_url = "https://tr-en.openfoodfacts.org/data-quality-errors" - json_content = { - "count": 129, - "tags": [ - { - "id": "en:ecoscore-production-system-no-label", - "known": 0, - "name": "ecoscore-production-system-no-label", - "products": 1848, - "url": f"{base_url}/ecoscore-production-system-no-label", - }, - { - "id": "en:no-packaging-data", - "known": 0, - "name": "no-packaging-data", - "products": 1788, - "url": f"{base_url}/no-packaging-data", - }, - { - "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "known": 0, - "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "products": 1778, - "url": ( - f"{base_url}/" "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown" - ), - }, - ], - } - - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory(expected_url, json_content=json_content), - ) - result = await KnowledgePanels( - facet="country", value="Turkey", country="Hungary" - ).data_quality_kp() - first_element = result["Quality"]["elements"][0] - first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) - expected_text = """ - - """ # noqa: E501 # allow long lines - # assert html separately to have better output in case of error - assert first_element["text_element"]["html"] == tidy_html(expected_text) - # now replace it for concision of output - first_element["text_element"]["html"] = "ok" - assert result == { - "Quality": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "ok", - "source_text": "Data-quality issues", - "source_url": "https://tr-en.openfoodfacts.org/data-quality-errors", - }, - } - ], - "title_element": {"title": "Data-quality issues related to Turkey "}, - } - } - - -async def test_data_quality_kp_with_one_facet_and_value(monkeypatch): - expected_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors.json" - base_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors" - json_content = { - "count": 181, - "tags": [ - { - "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "known": 0, - "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "products": 7898, - "url": ( - f"{base_url}/" "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown" - ), - }, - { - "id": "en:ecoscore-production-system-no-label", - "known": 0, - "name": "ecoscore-production-system-no-label", - "products": 7883, - "url": f"{base_url}/ecoscore-production-system-no-label", - }, - { - "id": "en:no-packaging-data", - "known": 0, - "name": "no-packaging-data", - "products": 6406, - "url": f"{base_url}/no-packaging-data", - }, - ], - } - - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory(expected_url, json_content=json_content), - ) - result = await KnowledgePanels(facet="brand", value="lidl").data_quality_kp() - first_element = result["Quality"]["elements"][0] - first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) - expected_text = """ - - """ # noqa: E501 # allow long lines - # assert html separately to have better output in case of error - assert first_element["text_element"]["html"] == tidy_html(expected_text) - # now replace it for concision of output - first_element["text_element"]["html"] = "ok" - assert result == { - "Quality": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "ok", - "source_text": "Data-quality issues", - "source_url": "https://world.openfoodfacts.org/brand/lidl/" - + "data-quality-errors", - }, - } - ], - "title_element": {"title": "Data-quality issues related to brand lidl"}, - } - } - - -async def test_data_quality_kp_with_one_facet_and_value_plural_facet(monkeypatch): - expected_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors.json" - base_url = "https://world.openfoodfacts.org/brand/lidl/data-quality-errors" - json_content = { - "count": 181, - "tags": [ - { - "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "known": 0, - "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "products": 7898, - "url": ( - f"{base_url}/" "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown" - ), - }, - { - "id": "en:ecoscore-production-system-no-label", - "known": 0, - "name": "ecoscore-production-system-no-label", - "products": 7883, - "url": f"{base_url}/ecoscore-production-system-no-label", - }, - { - "id": "en:no-packaging-data", - "known": 0, - "name": "no-packaging-data", - "products": 6406, - "url": f"{base_url}/no-packaging-data", - }, - ], - } - - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory(expected_url, json_content=json_content), - ) - result = await KnowledgePanels(facet="brands", value="lidl").data_quality_kp() - first_element = result["Quality"]["elements"][0] - first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) - expected_text = """ - - """ # noqa: E501 # allow long lines - # assert html separately to have better output in case of error - assert first_element["text_element"]["html"] == tidy_html(expected_text) - # now replace it for concision of output - first_element["text_element"]["html"] = "ok" - assert result == { - "Quality": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "ok", - "source_text": "Data-quality issues", - "source_url": "https://world.openfoodfacts.org/brand/lidl/" - + "data-quality-errors", - }, - } - ], - "title_element": {"title": "Data-quality issues related to brand lidl"}, - } - } - - -async def test_data_quality_kp_with_all_tags(monkeypatch): - expected_url = ( - "https://world.openfoodfacts.org/category/beers/brand/budweiser/data-quality-errors.json" - ) - json_content = { - "count": 24, - "tags": [ - { - "id": "en:alcoholic-beverages-category-without-alcohol-value", - "known": 0, - "name": "alcoholic-beverages-category-without-alcohol-value", - "products": 13, - "url": "https://world.openfoodfacts.org/category/beers/data-quality-errors/alcoholic-beverages-category-without-alcohol-value", # noqa: E501 # allow long lines - }, - { - "id": "en:ecoscore-production-system-no-label", - "known": 0, - "name": "ecoscore-production-system-no-label", - "products": 13, - "url": "https://world.openfoodfacts.org/category/beers/data-quality-errors/ecoscore-production-system-no-label", # noqa: E501 # allow long lines - }, - { - "id": "en:ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "known": 0, - "name": "ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", - "products": 12, - "url": "https://world.openfoodfacts.org/category/beers/data-quality-errors/ecoscore-origins-of-ingredients-origins-are-100-percent-unknown", # noqa: E501 # allow long lines - }, - ], - } - - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory(expected_url, json_content=json_content), - ) - result = await KnowledgePanels( - facet="category", value="beers", sec_facet="brand", sec_value="budweiser" - ).data_quality_kp() - first_element = result["Quality"]["elements"][0] - first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) - expected_text = """ - - """ # noqa: E501 # allow long lines - # assert html separately to have better output in case of error - assert first_element["text_element"]["html"] == tidy_html(expected_text) - # now replace it for concision of output - first_element["text_element"]["html"] = "ok" - assert result == { - "Quality": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "ok", - "source_text": "Data-quality issues", - "source_url": "https://world.openfoodfacts.org/category/beers/brand/budweiser/data-quality-errors", # noqa: E501 - }, - } - ], - "title_element": { - "title": "Data-quality issues related to category beers brand budweiser" - }, - } - } - - -async def test_last_edits_kp_with_one_facet_and_value(monkeypatch): - expected_url = "https://hu-en.openfoodfacts.org/api/v2/search" - expected_kwargs = { - "params": { - "fields": "product_name,code,last_editor,last_edit_dates_tags", - "sort_by": "last_modified_t", - "vitamins_tags_en": "vitamin-k", - } - } - json_content = { - "count": 1, - "page": 1, - "page_count": 1, - "page_size": 24, - "products": [ - { - "code": "0715235567418", - "last_edit_dates_tags": ["2022-02-10", "2022-02", "2022"], - "last_editor": "packbot", - "product_name": "Tiqle Sticks Strawberry taste", - } - ], - } - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory( - expected_url, - expected_kwargs, - json_content, - ), - ) - result = await KnowledgePanels( - facet="vitamin", value="vitamin-k", country="hungary" - ).last_edits_kp() - first_element = result["LastEdits"]["elements"][0] - first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) - last_edits_text = """ - - """ - # assert html separately to have better output in case of error - assert first_element["text_element"]["html"] == tidy_html(last_edits_text) - # now replace it for concision of output - first_element["text_element"]["html"] = "ok" - assert result == { - "LastEdits": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "ok", - "source_text": "Last-edits", - "source_url": "https://hu-en.openfoodfacts.org/vitamin/vitamin-k?sort_by=last_modified_t", # noqa: E501 - }, - } - ], - "title_element": {"title": "last-edits related to hungary vitamin vitamin-k"}, - } - } - - -async def test_last_edits_kp_with_one_facet_and_value_plural_facet(monkeypatch): - expected_url = "https://hu-en.openfoodfacts.org/api/v2/search" - expected_kwargs = { - "params": { - "fields": "product_name,code,last_editor,last_edit_dates_tags", - "sort_by": "last_modified_t", - "vitamins_tags_en": "vitamin-k", - } - } - json_content = { - "count": 1, - "page": 1, - "page_count": 1, - "page_size": 24, - "products": [ - { - "code": "0715235567418", - "last_edit_dates_tags": ["2022-02-10", "2022-02", "2022"], - "last_editor": "packbot", - "product_name": "Tiqle Sticks Strawberry taste", - } - ], - } - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory( - expected_url, - expected_kwargs, - json_content, - ), - ) - result = await KnowledgePanels( - facet="vitamins", value="vitamin-k", country="hungary" - ).last_edits_kp() - first_element = result["LastEdits"]["elements"][0] - first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) - last_edits_text = """ - - """ - # assert html separately to have better output in case of error - assert first_element["text_element"]["html"] == tidy_html(last_edits_text) - # now replace it for concision of output - first_element["text_element"]["html"] = "ok" - assert result == { - "LastEdits": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "ok", - "source_text": "Last-edits", - "source_url": "https://hu-en.openfoodfacts.org/vitamin/vitamin-k?sort_by=last_modified_t", # noqa: E501 - }, - } - ], - "title_element": {"title": "last-edits related to hungary vitamin vitamin-k"}, - } - } - - -async def test_last_edits_kp_with_all_tags(monkeypatch): - expected_url = "https://fr-en.openfoodfacts.org/api/v2/search" - expected_kwargs = { - "params": { - "fields": "product_name,code,last_editor,last_edit_dates_tags", - "sort_by": "last_modified_t", - "brands_tags_en": "nestle", - "categories_tags_en": "coffees", - } - } - json_content = { - "count": 116, - "page": 1, - "page_count": 24, - "page_size": 24, - "products": [ - { - "code": "7613036271868", - "last_edit_dates_tags": ["2022-08-31", "2022-08", "2022"], - "last_editor": "org-nestle-france", - "product_name": "Capsules NESCAFE Dolce Gusto Cappuccino Extra Crema 16 Capsules", - }, - { - "code": "7613032655495", - "last_edit_dates_tags": ["2022-08-30", "2022-08", "2022"], - "last_editor": "feat", - "product_name": "RICORE Original, Café & Chicorée, Boîte 260g", - }, - { - "code": "7613036303521", - "last_edit_dates_tags": ["2022-08-28", "2022-08", "2022"], - "last_editor": "feat", - "product_name": "Ricoré", - }, - { - "code": "3033710072927", - "last_edit_dates_tags": ["2022-08-28", "2022-08", "2022"], - "last_editor": "org-nestle-france", - "product_name": "NESCAFÉ NES, Café Soluble, Boîte de 25 Sticks (2g chacun)", - }, - { - "code": "3033710076017", - "last_edit_dates_tags": ["2022-08-28", "2022-08", "2022"], - "last_editor": "org-nestle-france", - "product_name": "NESCAFÉ SPECIAL FILTRE L'Original, Café Soluble, Boîte de 25 Sticks", # noqa: E501 # allow long lines - }, - { - "code": "3033710074624", - "last_edit_dates_tags": ["2022-08-28", "2022-08", "2022"], - "last_editor": "org-nestle-france", - "product_name": "NESCAFÉ SPECIAL FILTRE Décaféiné, Café Soluble, Flacon de 200g", - }, - { - "code": "7613034056122", - "last_edit_dates_tags": ["2022-08-28", "2022-08", "2022"], - "last_editor": "org-nestle-france", - "product_name": "NESCAFÉ SPECIAL FILTRE L'Original, Café Soluble, Recharge de 150g", - }, - { - "code": "3033710074525", - "last_edit_dates_tags": ["2022-08-28", "2022-08", "2022"], - "last_editor": "org-nestle-france", - "product_name": "NESCAFÉ SPECIAL FILTRE L'Original Flacon de 200g", - }, - { - "code": "3033710074518", - "last_edit_dates_tags": ["2022-08-28", "2022-08", "2022"], - "last_editor": "org-nestle-france", - }, - { - "code": "7891000300602", - "last_edit_dates_tags": ["2022-08-27", "2022-08", "2022"], - "last_editor": "5m4u9", - "product_name": "Original", - }, - ], - } - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory( - expected_url, - expected_kwargs, - json_content, - ), - ) - result = await KnowledgePanels( - facet="brand", - value="nestle", - sec_facet="category", - sec_value="coffees", - country="france", - ).last_edits_kp() - first_element = result["LastEdits"]["elements"][0] - first_element["text_element"]["html"] = tidy_html(first_element["text_element"]["html"]) - last_edits_text = """ - - """ # noqa: E501 # allow long lines - # assert html separately to have better output in case of error - assert first_element["text_element"]["html"] == tidy_html(last_edits_text) - # now replace it for concision of output - first_element["text_element"]["html"] = "ok" - assert result == { - "LastEdits": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "ok", - "source_text": "Last-edits", - "source_url": "https://fr-en.openfoodfacts.org/brand/nestle/category/coffees?sort_by=last_modified_t", # noqa: E501 - }, - } - ], - "title_element": { - "title": "last-edits related to france brand nestle category coffees" - }, - } - } - - -async def test_wikidata_kp_no_value(): - # wikidata only fetched if there is a value - result = await KnowledgePanels(facet="category").wikidata_kp() - assert result is None - - -async def test_wikidata_no_wikidata_property(monkeypatch): - # first mock the call to open food facts (to get the wikidata property) - expected_url = "https://world.openfoodfacts.org/api/v2/taxonomy" - expected_kwargs = { - "params": { - "tagtype": "categories", - "fields": "wikidata", - "tags": "fr:fitou", - } - } - # no wikidata entry ! - json_content = {"fr:fitou": {"parents": []}} - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory( - expected_url, - expected_kwargs, - json_content, - ), - ) - result = await KnowledgePanels(facet="category", value="fr:fitou").wikidata_kp() - assert result is None - - -async def test_wikidata_kp(monkeypatch): - # first mock the call to open food facts (to get the wikidata property) - expected_url = "https://world.openfoodfacts.org/api/v2/taxonomy" - expected_kwargs = { - "params": { - "tagtype": "categories", - "fields": "wikidata", - "tags": "fr:fitou", - } - } - json_content = {"fr:fitou": {"parents": [], "wikidata": {"en": "Q470974"}}} - monkeypatch.setattr( - aiohttp.ClientSession, - "get", - mock_async_get_factory( - expected_url, - expected_kwargs, - json_content, - ), - ) - # then mock the call to wikidata - # fake entity mimicks the Entity object from wikidata library - image_url = ( - "https://upload.wikimedia.org/wikipedia/commons/d/d6/" - "Paziols_%28France%29_Vue_du_village.jpg" - ) - fake_entity = { - "description": {"en": "French wine appellation", "fr": "région viticole"}, - "label": {"en": "Fitou AOC", "fr": "Fitou"}, - wikidata_props.image_prop: DictAttr(image_url=image_url), - wikidata_props.OSM_prop: "2727716", - wikidata_props.INAO_prop: "6159", - "attributes": { - "sitelinks": { - "enwiki": {"url": "http://en.wikipedia.org/wiki/Fitou_AOC"}, - "frwiki": {"url": "http://fr.wikipedia.org/wiki/Fitou_AOC"}, - } - }, - } - monkeypatch.setattr( - wikidata.client.Client, - "get", - mock_wikidata_get("Q470974", fake_entity), - ) - # run the test - result = await KnowledgePanels(facet="category", value="fr:fitou").wikidata_kp() - plural_result = await KnowledgePanels(facet="categories", value="fr:fitou").wikidata_kp() - image_thumb = ( - "https://upload.wikimedia.org/wikipedia/commons/thumb/d/d6/" - "Paziols_%28France%29_Vue_du_village.jpg/320px-thumbnail.jpg" - ) - clean_html = ( - f"

wikidata image

" - "
  • wikipedia" - "
    • " - "
  • OpenSteetMap Relation" - "
    • " - "
  • INAO relation" - "
    • " - "" - ) - expected_result = { - "WikiData": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "" - ) - expected_result_fr = { - "WikiData": { - "elements": [ - { - "element_type": "text", - "text_element": { - "html": "" + ) + expected_result = { + "WikiData": { + "elements": [ + { + "element_type": "text", + "text_element": { + "html": "" + ) + expected_result_fr = { + "WikiData": { + "elements": [ + { + "element_type": "text", + "text_element": { + "html": "