From ac6b91265db91fc80485839fee06567e8deae5f2 Mon Sep 17 00:00:00 2001 From: benbenben2 <110821832+benbenben2@users.noreply.github.com> Date: Sun, 14 Jan 2024 11:33:19 +0100 Subject: [PATCH] feat: dq_category_specific_ingredient_percent_2 (#9606) dq_category_specific_ingredient_percent_2 --- lib/ProductOpener/DataQualityFood.pm | 56 +++++++++++ taxonomies/categories.txt | 8 ++ taxonomies/data_quality.txt | 8 ++ tests/unit/dataqualityfood.t | 137 +++++++++++++++++++++++++++ 4 files changed, 209 insertions(+) diff --git a/lib/ProductOpener/DataQualityFood.pm b/lib/ProductOpener/DataQualityFood.pm index 1e46b944afaab..dded1baaa3faa 100644 --- a/lib/ProductOpener/DataQualityFood.pm +++ b/lib/ProductOpener/DataQualityFood.pm @@ -2391,6 +2391,62 @@ sub check_labels ($product_ref) { "en:high-in-omega-3-label-claim-but-ala-or-sum-of-epa-and-dha-below-limitation"); } } + + # In EU, compare categories and regulations + # https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A02001L0113-20131118 + # my %spread_categories_regulation = ( + # europe => { + # "en:jams" => "35", + + # some categories have mininal amount content required by regulations + # $expected_minimal_amount_specific_ingredients = "en:fruit, 35, en:eu" + my ($expected_minimal_amount_specific_ingredients, $category_id) + = get_inherited_property_from_categories_tags($product_ref, "expected_minimal_amount_specific_ingredients:en"); + + # convert as a list, in case there are more than a countries having regulations + my @expected_minimal_amount_specific_ingredients_list = split /;/, $expected_minimal_amount_specific_ingredients; + foreach + my $expected_minimal_amount_specific_ingredients_element (@expected_minimal_amount_specific_ingredients_list) + { + # split on ", " to extract ingredient id, quantity in g and country + my ($specific_ingredient_id, $quantity_threshold, $country) = split /, /, + $expected_minimal_amount_specific_ingredients_element; + + if ( + (defined $specific_ingredient_id) + and (defined $quantity_threshold) + and (defined $country) + and + ((($country eq "en:eu") and ($european_product == 1)) or (has_tag(($product_ref, "countries", $country)))) + ) + { + my $specific_ingredient_quantity; + if (defined $product_ref->{specific_ingredients}) { + foreach my $specific_ingredient ($product_ref->{specific_ingredients}[0]) { + if ( (defined $specific_ingredient->{id}) + and (defined $specific_ingredient->{quantity_g}) + and ($specific_ingredient->{id} eq $specific_ingredient_id)) + { + $specific_ingredient_quantity = $specific_ingredient->{quantity_g}; + } + } + } + + if (defined $specific_ingredient_quantity) { + if ($specific_ingredient_quantity < $quantity_threshold) { + add_tag($product_ref, "data_quality_errors", + "en:specific-ingredient-" + . substr($specific_ingredient_id, 3) + . "-quantity-is-below-the-minimum-value-of-$quantity_threshold-for-category-" + . substr($category_id, 3)); + } + } + else { + add_tag($product_ref, "data_quality_info", "en:missing-specific-ingredient-for-this-category"); + } + + } + } return; } diff --git a/taxonomies/categories.txt b/taxonomies/categories.txt index e183580a718c3..b29bbb312c9ae 100644 --- a/taxonomies/categories.txt +++ b/taxonomies/categories.txt @@ -65,6 +65,12 @@ stopwords:de:und,mit,von # And for categories where we are certain there is 100% fruits/vegetables/legumes # we put nutriscore_category_override_for_fruits_vegetables_legumes:en: 100 +# For categories for which a minimum amount of specific ingredients is required in some countries +# we put expected_minimal_amount_specific_ingredients:en followed by the specific ingredient, +# the quantity in g and the country where it applis. +# additional countries can be added separated by a semi colon (";"). Example for en:fruit, 35g, in EU and en:fruit, 35g in en:other-country: +# expected_minimal_amount_specific_ingredients:en: en:fruit, 35, en:eu; en:fruit, 35, en:other-country + en:Artisan products ca:Prouctes artesans de:Handgefertigte Produkte, Artisanale Produkte, Handgemachte Produkte @@ -92485,6 +92491,7 @@ ciqual_proxy_food_code:en:31024 ciqual_proxy_food_name:en:Jam, strawberry ciqual_proxy_food_name:fr:Confiture de fraise (extra ou classique) intake24_category_code:en:JAMS +expected_minimal_amount_specific_ingredients:en: en:fruit, 35, en:eu ["en:jams"], + countries_tags => ["en:slovenia",], +}; +ProductOpener::DataQuality::check_quality($product_ref); +ok(has_tag($product_ref, 'data_quality', 'en:missing-specific-ingredient-for-this-category'), + 'specific ingredients missing') + or diag explain $product_ref; +## missing specific ingredients for fruit +$product_ref = { + categories_tags => ["en:jams"], + countries_tags => ["en:slovenia",], + specific_ingredients => [ + { + id => "en:other", + ingredient => "other", + quantity => "50 g", + quantity_g => 50, + text => "other", + }, + ] +}; +ProductOpener::DataQuality::check_quality($product_ref); +ok(has_tag($product_ref, 'data_quality', 'en:missing-specific-ingredient-for-this-category'), + 'specific ingredients but en:fruit missing') + or diag explain $product_ref; +## specific ingredients for fruit ok +$product_ref = { + categories_tags => ["en:jams"], + countries_tags => ["en:slovenia",], + specific_ingredients => [ + { + id => "en:fruit", + ingredient => "fruit", + quantity => "50 g", + quantity_g => 50, + text => "Prepared with 50g fruit per 100g", + }, + ] +}; +ProductOpener::DataQuality::check_quality($product_ref); +ok(!has_tag($product_ref, 'data_quality', 'en:missing-fruit-content-for-jams-or-jellies'), + 'specific ingredients with en:fruit ok') + or diag explain $product_ref; +ok( + !has_tag( + $product_ref, 'data_quality', + 'en:specific-ingredient-fruit-quantity-is-below-the-minimum-value-of-35-for-category-jams' + ), + 'en:fruit content ok' +) or diag explain $product_ref; +## specific ingredients for fruit is given but content is too small +$product_ref = { + categories_tags => ["en:jams"], + countries_tags => ["en:slovenia",], + specific_ingredients => [ + { + id => "en:fruit", + ingredient => "fruit", + quantity => "5 g", + quantity_g => 5, + text => "Prepared with 5g fruit per 100g", + }, + ] +}; +ProductOpener::DataQuality::check_quality($product_ref); +ok(!has_tag($product_ref, 'data_quality', 'en:missing-fruit-content-for-jams-or-jellies'), + 'specific ingredients with en:fruit ok') + or diag explain $product_ref; +ok( + has_tag( + $product_ref, 'data_quality', + 'en:specific-ingredient-fruit-quantity-is-below-the-minimum-value-of-35-for-category-jams' + ), + 'en:fruit content too small' +) or diag explain $product_ref; +## specific ingredients for fruit is given but content is too small with more specific category +$product_ref = { + categories_tags => ["en:jams", "en:redcurrants-jams"], + countries_tags => ["en:slovenia",], + specific_ingredients => [ + { + id => "en:fruit", + ingredient => "fruit", + quantity => "10 g", + quantity_g => 10, + text => "Prepared with 10 fruit per 100g", + }, + ] +}; +ProductOpener::DataQuality::check_quality($product_ref); +ok( + !has_tag( + $product_ref, 'data_quality', + 'en:specific-ingredient-fruit-quantity-is-below-the-minimum-value-of-35-for-category-jams' + ), + 'en:fruit content too small for jam but has more specific category with smaller threshold' +) or diag explain $product_ref; +ok( + has_tag( + $product_ref, 'data_quality', + 'en:specific-ingredient-fruit-quantity-is-below-the-minimum-value-of-25-for-category-redcurrants-jams' + ), + 'en:fruit content too small' +) or diag explain $product_ref; +## specific ingredients for fruit is given but content is too small for jams but high enough for more specific category +$product_ref = { + categories_tags => ["en:jams", "en:redcurrants-jams"], + countries_tags => ["en:slovenia",], + specific_ingredients => [ + { + id => "en:fruit", + ingredient => "fruit", + quantity => "30 g", + quantity_g => 30, + text => "Prepared with 30 fruit per 100g", + }, + ] +}; +ProductOpener::DataQuality::check_quality($product_ref); +ok( + !has_tag( + $product_ref, 'data_quality', + 'en:specific-ingredient-fruit-quantity-is-below-the-minimum-value-of-35-for-category-jams' + ), + 'en:fruit content too small for jam but has more specific category with smaller threshold' +) or diag explain $product_ref; +ok( + !has_tag( + $product_ref, 'data_quality', + 'en:specific-ingredient-fruit-quantity-is-below-the-minimum-value-of-25-for-category-redcurrants-jams' + ), + 'en:fruit content too small' +) or diag explain $product_ref; + done_testing();