From 8ce0abcd21c19d05fa613e96c38d49a1d94abe73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Tue, 29 Aug 2023 19:15:09 +0200 Subject: [PATCH] feat: New Nutri-Score formula (v2) for food (2022) and beverages (2023) - ready for review (#8832) --- lib/ProductOpener/Food.pm | 541 ++++++++++++++---- lib/ProductOpener/Import.pm | 2 +- lib/ProductOpener/Nutriscore.pm | 490 ++++++++++++++-- lib/ProductOpener/Products.pm | 5 +- po/tags/en.po | 29 + po/tags/tags.pot | 29 + scripts/aggregate_ingredients.pl | 20 - scripts/extract_individual_ingredients.pl | 20 - scripts/import_fleurymichon.pl | 83 +-- scripts/import_systemeu.pl | 132 +---- scripts/list_ingredients.pl | 20 - scripts/revert_changes_from_user.pl | 10 - scripts/update_all_products.pl | 17 +- stop_words.txt | 3 +- taxonomies/categories.txt | 5 +- .../get-existing-product.json | 63 +- .../get-fields-all-knowledge-panels.json | 63 +- .../api_v2_product_read/get-fields-all.json | 63 +- ...attribute-groups-all-knowledge-panels.json | 63 +- .../api_v2_product_read/get-fields-raw.json | 63 +- .../get-product-auth-good-password.json | 63 +- ...uct-ingredients-text-without-language.json | 62 +- .../api_v2_product_write/get-product.json | 63 +- .../get-existing-product.json | 63 +- .../get-fields-all-knowledge-panels.json | 63 +- .../api_v3_product_read/get-fields-all.json | 63 +- ...attribute-groups-all-knowledge-panels.json | 63 +- .../api_v3_product_read/get-fields-raw.json | 63 +- ...redients-categories-to-get-nutriscore.json | 3 - .../patch-packagings-fr-fields.json | 4 +- .../patch-packagings-quantity-and-weight.json | 4 +- ...kagings-weights-as-strings-with-units.json | 4 +- .../patch-packagings-weights-as-strings.json | 4 +- .../patch-request-fields-all.json | 62 +- .../patch-weight-as-number-or-string.json | 4 +- .../products/3250390017165.json | 62 +- .../products/3250390020745.json | 62 +- .../products/3250390020806.json | 62 +- .../products/3250390020998.json | 62 +- .../products/3250390021001.json | 62 +- .../products/3250390021469.json | 62 +- .../products/3250390021544.json | 62 +- .../products/3250390021568.json | 62 +- .../products/3250390021926.json | 62 +- .../products/3250390024781.json | 62 +- .../products/3250390024804.json | 62 +- .../products/3250390024842.json | 62 +- .../products/3250390024866.json | 62 +- .../products/3250390025399.json | 62 +- .../products/3250390025863.json | 62 +- .../products/3250390026044.json | 62 +- .../products/3250390026648.json | 62 +- .../products/3250390026754.json | 62 +- .../test/products/3270190128403.json | 66 ++- .../test/products/4270190128403.json | 64 ++- .../test/products/5270190128403.json | 110 ++-- .../test/products/7270190128403.json | 62 +- .../export_more_fields/export_more_fields.csv | 20 +- .../export_more_fields/rows/26281742.json | 2 +- .../export_more_fields/rows/29161690.json | 2 +- .../rows/3173990027337.json | 2 +- .../rows/3250392332105.json | 2 +- .../rows/3259330020135.json | 2 +- .../rows/3760178254021.json | 2 +- .../rows/3770013801303.json | 2 +- .../export_more_fields/rows/77000001.json | 2 +- .../export_more_fields/rows/80650904.json | 2 +- .../rows/8712423020221.json | 2 +- .../3003004006001.json | 62 +- .../3003004006002.json | 62 +- .../3003004006003.json | 62 +- .../3003004006004.json | 62 +- .../3003004006005.json | 62 +- .../3003004006006.json | 62 +- .../3003004006007.json | 62 +- .../import_csv_file/test/2003004006001.json | 62 +- .../import_csv_file/test/2003004006002.json | 62 +- .../import_csv_file/test/2003004006003.json | 62 +- .../import_csv_file/test/2003004006004.json | 62 +- .../import_csv_file/test/2003004006005.json | 62 +- .../import_csv_file/test/2003004006006.json | 62 +- .../import_csv_file/test/2003004006007.json | 62 +- .../page_crawler/get-robots-txt-ch-it.txt | 3 + .../get-robots-txt-fr-pro-platform.txt | 3 + .../page_crawler/get-robots-txt-fr.txt | 3 + .../get-robots-txt-world-pro-platform.txt | 3 + .../page_crawler/get-robots-txt-world.txt | 3 + ...ed-protected-product-api-v2-moderator.json | 63 +- .../get-edited-protected-product-api-v2.json | 63 +- ...-protected-product-web-form-moderator.json | 63 +- ...get-edited-protected-product-web-form.json | 63 +- ...get-edited-unprotected-product-api-v2.json | 63 +- ...t-edited-unprotected-product-web-form.json | 63 +- .../search_v1/search-no-filter.json | 416 +++++++++++++- ...ies-without-ingredients-from-palm-oil.json | 103 +++- ...rch-without-ingredients-from-palm-oil.json | 292 +++++++++- .../attributes/en-attributes.json | 100 +++- .../en-ecoscore-score-at-20-threshold.json | 62 +- .../attributes/en-maybe-vegan.json | 63 +- .../attributes/en-no-ingredients.json | 62 +- .../attributes/en-nova-groups-markers.json | 63 +- .../attributes/en-nutriscore.json | 100 +++- .../attributes/en-unknown-ingredients.json | 63 +- .../attributes/fr-palm-kernel-fat.json | 63 +- .../attributes/fr-palm-oil-free.json | 63 +- .../attributes/fr-palm-oil.json | 63 +- .../attributes/fr-vegetable-oils.json | 63 +- ...-percent-sugar-and-unknown-ingredient.json | 67 ++- .../beverage-with-80-percent-milk.json | 106 +++- .../nutriscore/breakfast-cereals.json | 107 +++- .../cocoa-and-chocolate-powders.json | 108 +++- .../nutriscore/colza-oil.json | 113 +++- .../nutriscore/cookies.json | 106 +++- .../dairy-drink-with-80-percent-milk.json | 106 +++- ...-drink-with-less-than-80-percent-milk.json | 108 +++- .../nutriscore/dairy-drinks-without-milk.json | 108 +++- .../en-apple-estimated-nutrients.json | 105 +++- .../nutriscore/en-beers-category.json | 69 ++- ...orange-juice-category-and-ingredients.json | 108 +++- .../nutriscore/en-orange-juice-category.json | 108 +++- .../en-sugar-estimated-nutrients.json | 104 +++- .../flavored-spring-water-no-nutrition.json | 66 ++- .../flavored-spring-with-nutrition.json | 108 +++- .../nutriscore/fr-gaspacho.json | 108 +++- .../nutriscore/fr-orange-nectar-0-fat.json | 109 +++- .../nutriscore/milk.json | 106 +++- .../nutriscore/mushrooms.json | 108 +++- .../nutriscore/olive-oil.json | 113 +++- .../nutriscore/spring-water-no-nutrition.json | 100 +++- .../sunflower-oil-no-sugar-no-sat-fat.json | 78 ++- .../nutriscore/sunflower-oil-no-sugar.json | 115 +++- .../nutriscore/sunflower-oil.json | 113 +++- .../nutriscore/walnut-oil.json | 113 +++- ...-percent-sugar-and-unknown-ingredient.json | 193 +++++++ .../beverage-with-80-percent-milk.json | 268 +++++++++ .../nutriscore_2023/breakfast-cereals.json | 216 +++++++ .../cocoa-and-chocolate-powders.json | 206 +++++++ .../nutriscore_2023/colza-oil.json | 229 ++++++++ .../nutriscore_2023/cookies.json | 210 +++++++ .../dairy-drink-with-80-percent-milk.json | 276 +++++++++ ...-drink-with-less-than-80-percent-milk.json | 277 +++++++++ .../dairy-drinks-without-milk.json | 318 ++++++++++ .../en-apple-estimated-nutrients.json | 310 ++++++++++ .../nutriscore_2023/en-beers-category.json | 130 +++++ ...orange-juice-category-and-ingredients.json | 310 ++++++++++ .../en-orange-juice-category.json | 224 ++++++++ .../en-sugar-estimated-nutrients.json | 303 ++++++++++ .../flavored-spring-water-no-nutrition.json | 126 ++++ .../flavored-spring-with-nutrition.json | 216 +++++++ .../nutriscore_2023/fr-gaspacho.json | 443 ++++++++++++++ .../fr-orange-nectar-0-fat.json | 321 +++++++++++ .../nutriscore_2023/honey.json | 142 +++++ .../nutriscore_2023/milk.json | 258 +++++++++ .../nutriscore_2023/mushrooms.json | 276 +++++++++ .../nutriscore_2023/olive-oil.json | 231 ++++++++ .../spring-water-no-nutrition.json | 192 +++++++ .../sunflower-oil-no-sugar-no-sat-fat.json | 142 +++++ .../sunflower-oil-no-sugar.json | 229 ++++++++ .../nutriscore_2023/sunflower-oil.json | 229 ++++++++ .../nutriscore_2023/walnut-oil.json | 233 ++++++++ tests/unit/nutriscore.t | 17 +- tests/unit/nutriscore_2023.t | 463 +++++++++++++++ 162 files changed, 15462 insertions(+), 790 deletions(-) create mode 100644 tests/unit/expected_test_results/nutriscore_2023/94-percent-sugar-and-unknown-ingredient.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/beverage-with-80-percent-milk.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/breakfast-cereals.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/cocoa-and-chocolate-powders.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/colza-oil.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/cookies.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/dairy-drink-with-80-percent-milk.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/dairy-drink-with-less-than-80-percent-milk.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/dairy-drinks-without-milk.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/en-apple-estimated-nutrients.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/en-beers-category.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/en-orange-juice-category-and-ingredients.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/en-orange-juice-category.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/en-sugar-estimated-nutrients.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/flavored-spring-water-no-nutrition.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/flavored-spring-with-nutrition.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/fr-gaspacho.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/fr-orange-nectar-0-fat.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/honey.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/milk.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/mushrooms.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/olive-oil.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/spring-water-no-nutrition.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/sunflower-oil-no-sugar-no-sat-fat.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/sunflower-oil-no-sugar.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/sunflower-oil.json create mode 100644 tests/unit/expected_test_results/nutriscore_2023/walnut-oil.json create mode 100644 tests/unit/nutriscore_2023.t diff --git a/lib/ProductOpener/Food.pm b/lib/ProductOpener/Food.pm index b0f0849b8eed9..2b6d3e2eeb790 100644 --- a/lib/ProductOpener/Food.pm +++ b/lib/ProductOpener/Food.pm @@ -65,7 +65,7 @@ BEGIN { &is_fat_for_nutrition_score &compute_nutriscore - &compute_nutrition_score + &compute_nutriscore &compute_nova_group &compute_serving_size_data &compute_unknown_nutrients @@ -115,6 +115,9 @@ use URI::Escape::XS; use CGI qw/:cgi :form escapeHTML/; +use Data::DeepAccess qw(deep_set); +use Storable qw/dclone/; + use Log::Any qw($log); # Load nutrient stats for all categories and countries @@ -856,8 +859,8 @@ sub is_cheese_for_nutrition_score ($product_ref) { =head2 is_fat_for_nutrition_score( $product_ref ) -Determines if a product should be considered as fat for Nutri-Score computations, -based on the product categories. +Determines if a product should be considered as fat +for Nutri-Score (2021 version) computations, based on the product categories. =cut @@ -866,6 +869,60 @@ sub is_fat_for_nutrition_score ($product_ref) { return has_tag($product_ref, "categories", "en:fats"); } +=head2 is_fat_oil_nuts_seeds_for_nutrition_score( $product_ref ) + +Determines if a product should be considered as fats / oils / nuts / seeds +for Nutri-Score (2023 version) computations, based on the product categories. + +From the 2022 main algorithm report update FINAL: + +"This category includes fats and oils from plant or animal sources, including cream, margarines, +butters and oils (as the current situation). + +Additionally, the following products are included in this category, using the Harmonized System +Nomenclature1 codes: +- Nuts: 0801 0802 +- Processed nuts: 200811 200819 +- Ground nuts: 1202 +- Seeds: 1204 (linseed) 1206 (sunflower)1207 (other seeds) + +Of note chestnuts are excluded from the category." + +=cut + +sub is_fat_oil_nuts_seeds_for_nutrition_score ($product_ref) { + + if (has_tag($product_ref, "categories", "en:chestnuts")) { + return 0; + } + elsif (has_tag($product_ref, "categories", "en:fats") + or has_tag($product_ref, "categories", "en:creams") + or has_tag($product_ref, "categories", "en:seeds")) + { + return 1; + } + else { + my $hs_heading = get_inherited_property_from_categories_tags($product_ref, "wco_hs_code:en"); + + if (defined $hs_heading) { + my $hs_code = get_inherited_property_from_categories_tags($product_ref, "wco_hs_code:en"); + + if ( + ($hs_heading eq "08.01") or ($hs_heading eq "08.02") # nuts + or ($hs_code eq "2008.11") or ($hs_code eq "2008.19") # processed nuts + or ($hs_heading eq "12.02") # peanuts + or ($hs_heading eq "12.04") or ($hs_heading eq "12.06") or ($hs_heading eq "12.07") # nuts + ) + + { + return 1; + } + } + } + + return 0; +} + =head2 special_process_product ( $ingredients_ref ) Computes food groups, and whether a product is to be considered a beverage for the Nutri-Score. @@ -976,11 +1033,11 @@ sub compute_fruit_ratio ($product_ref, $prepared) { if (defined $product_ref->{nutriments}{"fruits-vegetables-nuts-dried" . $prepared . "_100g"}) { $fruits = 2 * $product_ref->{nutriments}{"fruits-vegetables-nuts-dried" . $prepared . "_100g"}; - push @{$product_ref->{misc_tags}}, "en:nutrition-fruits-vegetables-nuts-dried"; + add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-dried"); if (defined $product_ref->{nutriments}{"fruits-vegetables-nuts" . $prepared . "_100g"}) { $fruits += $product_ref->{nutriments}{"fruits-vegetables-nuts" . $prepared . "_100g"}; - push @{$product_ref->{misc_tags}}, "en:nutrition-fruits-vegetables-nuts"; + add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts"); } $fruits @@ -988,12 +1045,12 @@ sub compute_fruit_ratio ($product_ref, $prepared) { } elsif (defined $product_ref->{nutriments}{"fruits-vegetables-nuts" . $prepared . "_100g"}) { $fruits = $product_ref->{nutriments}{"fruits-vegetables-nuts" . $prepared . "_100g"}; - push @{$product_ref->{misc_tags}}, "en:nutrition-fruits-vegetables-nuts"; + add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts"); } elsif (defined $product_ref->{nutriments}{"fruits-vegetables-nuts-estimate" . $prepared . "_100g"}) { $fruits = $product_ref->{nutriments}{"fruits-vegetables-nuts-estimate" . $prepared . "_100g"}; $product_ref->{nutrition_score_warning_fruits_vegetables_nuts_estimate} = 1; - push @{$product_ref->{misc_tags}}, "en:nutrition-fruits-vegetables-nuts-estimate"; + add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-estimate"); } # Use the estimate from the ingredients list if we have one elsif ( @@ -1005,7 +1062,7 @@ sub compute_fruit_ratio ($product_ref, $prepared) { $fruits = $product_ref->{nutriments}{"fruits-vegetables-nuts-estimate-from-ingredients" . $prepared . "_100g"}; $product_ref->{nutrition_score_warning_fruits_vegetables_nuts_estimate_from_ingredients} = 1; $product_ref->{nutrition_score_warning_fruits_vegetables_nuts_estimate_from_ingredients_value} = $fruits; - push @{$product_ref->{misc_tags}}, "en:nutrition-fruits-vegetables-nuts-estimate-from-ingredients"; + add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-estimate-from-ingredients"); } else { # estimates by category of products. not exact values. it's important to distinguish only between the thresholds: 40, 60 and 80 @@ -1016,10 +1073,10 @@ sub compute_fruit_ratio ($product_ref, $prepared) { $product_ref->{nutrition_score_warning_fruits_vegetables_nuts_from_category} = $category_id; $product_ref->{nutrition_score_warning_fruits_vegetables_nuts_from_category_value} = $fruits_vegetables_nuts_by_category{$category_id}; - push @{$product_ref->{misc_tags}}, "en:nutrition-fruits-vegetables-nuts-from-category"; + add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-from-category"); my $category = $category_id; $category =~ s/:/-/; - push @{$product_ref->{misc_tags}}, "en:nutrition-fruits-vegetables-nuts-from-category-$category"; + add_tag($product_ref, "misc", "en:nutrition-fruits-vegetables-nuts-from-category-$category"); last; } } @@ -1028,17 +1085,17 @@ sub compute_fruit_ratio ($product_ref, $prepared) { if (not defined $fruits) { $fruits = 0; $product_ref->{nutrition_score_warning_no_fruits_vegetables_nuts} = 1; - push @{$product_ref->{misc_tags}}, "en:nutrition-no-fruits-vegetables-nuts"; + add_tag($product_ref, "misc", "en:nutrition-no-fruits-vegetables-nuts"); } } if ( (defined $product_ref->{nutrition_score_warning_no_fiber}) or (defined $product_ref->{nutrition_score_warning_no_fruits_vegetables_nuts})) { - push @{$product_ref->{misc_tags}}, "en:nutrition-no-fiber-or-fruits-vegetables-nuts"; + add_tag($product_ref, "misc", "en:nutrition-no-fiber-or-fruits-vegetables-nuts"); } else { - push @{$product_ref->{misc_tags}}, "en:nutrition-all-nutriscore-values-known"; + add_tag($product_ref, "misc", "en:nutrition-all-nutriscore-values-known"); } return $fruits; @@ -1136,69 +1193,111 @@ Ref to a mapping suitable to call compute_nutriscore_score_and_grade =cut -sub compute_nutriscore_data ($product_ref, $prepared, $nutriments_field) { +sub compute_nutriscore_data ($product_ref, $prepared, $nutriments_field, $version = "2021") { my $nutriments_ref = $product_ref->{$nutriments_field}; - # compute data - my $saturated_fat_ratio = saturated_fat_ratio($nutriments_ref, $prepared); - my $fruits = compute_fruit_ratio($product_ref, $prepared); - my $nutriscore_data = { - is_beverage => $product_ref->{nutrition_score_beverage}, - is_water => is_water_for_nutrition_score($product_ref), - is_cheese => is_cheese_for_nutrition_score($product_ref), - is_fat => is_fat_for_nutrition_score($product_ref), - - energy => $nutriments_ref->{"energy" . $prepared . "_100g"}, - sugars => $nutriments_ref->{"sugars" . $prepared . "_100g"}, - saturated_fat => $nutriments_ref->{"saturated-fat" . $prepared . "_100g"}, - saturated_fat_ratio => $saturated_fat_ratio, - sodium => $nutriments_ref->{"sodium" . $prepared . "_100g"} * 1000, # in mg, - - fruits_vegetables_nuts_colza_walnut_olive_oils => $fruits, - fiber => ( - (defined $nutriments_ref->{"fiber" . $prepared . "_100g"}) - ? $nutriments_ref->{"fiber" . $prepared . "_100g"} - : 0 - ), - proteins => $nutriments_ref->{"proteins" . $prepared . "_100g"}, - }; + my $nutriscore_data_ref; + + # The 2021 and 2023 version of the Nutri-Score need different nutrients + if ($version eq "2021") { + # fruits, vegetables, nuts, olive / rapeseed / walnut oils + my $fruits = compute_fruit_ratio($product_ref, $prepared); + + my $is_fat = is_fat_for_nutrition_score($product_ref); + + $nutriscore_data_ref = { + is_beverage => $product_ref->{nutrition_score_beverage}, + is_water => is_water_for_nutrition_score($product_ref), + is_cheese => is_cheese_for_nutrition_score($product_ref), + is_fat => $is_fat, + + energy => $nutriments_ref->{"energy" . $prepared . "_100g"}, + sugars => $nutriments_ref->{"sugars" . $prepared . "_100g"}, + saturated_fat => $nutriments_ref->{"saturated-fat" . $prepared . "_100g"}, + sodium => ( + (defined $nutriments_ref->{"sodium" . $prepared . "_100g"}) + ? $nutriments_ref->{"sodium" . $prepared . "_100g"} * 1000 + : undef + ), # in mg, + + fruits_vegetables_nuts_colza_walnut_olive_oils => $fruits, + fiber => ( + (defined $nutriments_ref->{"fiber" . $prepared . "_100g"}) + ? $nutriments_ref->{"fiber" . $prepared . "_100g"} + : 0 + ), + proteins => $nutriments_ref->{"proteins" . $prepared . "_100g"}, + }; + + if ($is_fat) { + # Add the fat and saturated fat / fat ratio + $nutriscore_data_ref->{fat} = $nutriments_ref->{"fat" . $prepared . "_100g"}; + $nutriscore_data_ref->{saturated_fat_ratio} = saturated_fat_ratio($nutriments_ref, $prepared); + } + } + else { + # TODO: needs to be replaced by "fruits, vegetables, legumes" + my $fruits = compute_fruit_ratio($product_ref, $prepared); + + my $is_fat_oil_nuts_seeds = is_fat_oil_nuts_seeds_for_nutrition_score($product_ref); + + $nutriscore_data_ref = { + is_beverage => $product_ref->{nutrition_score_beverage}, + is_water => is_water_for_nutrition_score($product_ref), + is_cheese => is_cheese_for_nutrition_score($product_ref), + is_fat_oil_nuts_seeds => $is_fat_oil_nuts_seeds, + + energy => $nutriments_ref->{"energy" . $prepared . "_100g"}, + sugars => $nutriments_ref->{"sugars" . $prepared . "_100g"}, + saturated_fat => $nutriments_ref->{"saturated-fat" . $prepared . "_100g"}, + salt => $nutriments_ref->{"salt" . $prepared . "_100g"}, + + fruits_vegetables_legumes => $fruits, + fiber => ( + (defined $nutriments_ref->{"fiber" . $prepared . "_100g"}) + ? $nutriments_ref->{"fiber" . $prepared . "_100g"} + : 0 + ), + proteins => $nutriments_ref->{"proteins" . $prepared . "_100g"}, + }; + + if ($is_fat_oil_nuts_seeds) { + # Add the fat and saturated fat / fat ratio + $nutriscore_data_ref->{fat} = $nutriments_ref->{"fat" . $prepared}; + $nutriscore_data_ref->{saturated_fat_ratio} = saturated_fat_ratio($nutriments_ref, $prepared); + # Compute the energy from saturates + $nutriscore_data_ref->{energy_from_saturated_fat} = $nutriscore_data_ref->{saturated_fat} * 37; + } + } # tweak data to take into account special cases # if sugar is undefined but carbohydrates is 0, set sugars to 0 if (sugar_0_because_of_carbohydrates_0($nutriments_ref, $prepared)) { - $nutriscore_data->{sugars} = 0; + $nutriscore_data_ref->{sugars} = 0; } # if saturated_fat is undefined but fat is 0, set saturated_fat to 0 # as well as saturated_fat_ratio if (saturated_fat_0_because_of_fat_0($nutriments_ref, $prepared)) { - $nutriscore_data->{saturated_fat} = 0; - $nutriscore_data->{saturated_fat_ratio} = 0; + $nutriscore_data_ref->{saturated_fat} = 0; + $nutriscore_data_ref->{saturated_fat_ratio} = 0; } - return $nutriscore_data; + return $nutriscore_data_ref; } -=head2 compute_nutrition_score( $product_ref ) - -Determines if we have enough data to compute the Nutri-Score (category + nutrition facts), -and if the Nutri-Score is applicable to the product the category. - -Populates the data structure needed to compute the Nutri-Score and computes it. +=head2 remove_nutriscore_fields ( $product_ref ) =cut -sub compute_nutrition_score ($product_ref) { - - # Initialize values - - $product_ref->{nutrition_score_debug} = ''; +sub remove_nutriscore_fields ($product_ref) { - # remove reference type fields from the product + # remove direct fields from the product remove_fields( $product_ref, [ + "nutriscore", "nutrition_score_warning_no_fiber", "nutrition_score_warning_fruits_vegetables_nuts_estimate", "nutrition_score_warning_fruits_vegetables_nuts_from_category", @@ -1213,11 +1312,14 @@ sub compute_nutrition_score ($product_ref) { "nutriscore_points", "nutrition_grade_fr", "nutrition_grades", - "nutrition_grades_tags" + "nutrition_grades_tags", + "nutriscore_tags", + "nutriscore_2021_tags", + "nutriscore_2023_tags", ] ); - # strip score-type fields from the product + # strip nutriments / score-type fields from the product remove_fields( $product_ref->{nutriments}, [ @@ -1229,49 +1331,45 @@ sub compute_nutrition_score ($product_ref) { ] ); - $product_ref->{misc_tags} = ["en:nutriscore-not-computed"]; + return; +} + +=head2 is_nutriscore_applicable_to_the_product_categories($product_ref) - my $prepared = ''; +Check that the product has a category, that we know if it is a beverage or not, +and that it is not in a category for which the Nutri-Score should not be computed +(e.g. food for babies) + +=head3 Return values + +=head4 $category_available - 0 or 1 + +=head4 $nutriscore_applicable - 0 or 1 + +=head4 $not_applicable_category - undef or category id + +=cut + +sub is_nutriscore_applicable_to_the_product_categories ($product_ref) { + + my $category_available = 1; + my $nutriscore_applicable = 1; + my $not_applicable_category = undef; # do not compute a score when we don't have a category if ((not defined $product_ref->{categories}) or ($product_ref->{categories} eq '')) { $product_ref->{"nutrition_grades_tags"} = ["unknown"]; $product_ref->{nutrition_score_debug} = "no score when the product does not have a category" . " - "; add_tag($product_ref, "misc", "en:nutriscore-missing-category"); + $category_available = 0; + $nutriscore_applicable = 0; } if (not defined $product_ref->{nutrition_score_beverage}) { $product_ref->{"nutrition_grades_tags"} = ["unknown"]; $product_ref->{nutrition_score_debug} = "did not determine if it was a beverage" . " - "; add_tag($product_ref, "misc", "en:nutriscore-beverage-status-unknown"); - } - - # do not compute a score for dehydrated products to be rehydrated (e.g. dried soups, powder milk) - # unless we have nutrition data for the prepared product - # same for en:chocolate-powders, en:dessert-mixes and en:flavoured-syrups - - foreach my $category_tag ( - "en:dried-products-to-be-rehydrated", "en:cocoa-and-chocolate-powders", - "en:dessert-mixes", "en:flavoured-syrups", - "en:instant-beverages" - ) - { - - if (has_tag($product_ref, "categories", $category_tag)) { - - if ((defined $product_ref->{nutriments}{"energy_prepared_100g"})) { - $product_ref->{nutrition_score_debug} - = "using prepared product data for category $category_tag" . " - "; - $prepared = '_prepared'; - } - else { - $product_ref->{"nutrition_grades_tags"} = ["unknown"]; - $product_ref->{nutrition_score_debug} - = "no score for category $category_tag without data for prepared product" . " - "; - add_tag($product_ref, "misc", "en:nutriscore-missing-prepared-nutrition-data"); - } - last; - } + $nutriscore_applicable = 0; } # do not compute a score for coffee, tea etc. except ice teas etc. @@ -1297,12 +1395,71 @@ sub compute_nutrition_score ($product_ref) { add_tag($product_ref, "misc", "en:nutriscore-not-applicable"); $product_ref->{nutrition_score_debug} = "no nutriscore for category $category_id" . " - "; $product_ref->{nutriscore_data} = {nutriscore_not_applicable_for_category => $category_id}; + $nutriscore_applicable = 0; + $not_applicable_category = $category_id; last; } } } } + return ($category_available, $nutriscore_applicable, $not_applicable_category); +} + +=head2 check_availability_of_nutrients_needed_for_nutriscore ($product_ref) + +Check that we know or can estimate the nutrients needed to compute the Nutri-Score of the product. + +=head3 Return values + +=head4 $nutrients_available 0 or 1 + +=head4 $prepared "" or "_prepared" + +Suffix to indicate if the Nutri-Score should be computed on prepared values + +=head4 $nutriments_field "nutriments" or "nutriments_estimated" + +Indicates which nutrients fields were used to compute the Nutri-Score. + +=cut + +sub check_availability_of_nutrients_needed_for_nutriscore ($product_ref) { + + my $nutrients_available = 1; + + # do not compute a score for dehydrated products to be rehydrated (e.g. dried soups, powder milk) + # unless we have nutrition data for the prepared product + # same for en:chocolate-powders, en:dessert-mixes and en:flavoured-syrups + + my $prepared = ''; + + foreach my $category_tag ( + "en:dried-products-to-be-rehydrated", "en:cocoa-and-chocolate-powders", + "en:dessert-mixes", "en:flavoured-syrups", + "en:instant-beverages" + ) + { + + if (has_tag($product_ref, "categories", $category_tag)) { + + if ((defined $product_ref->{nutriments}{"energy_prepared_100g"})) { + $product_ref->{nutrition_score_debug} + = "using prepared product data for category $category_tag" . " - "; + $prepared = '_prepared'; + add_tag($product_ref, "misc", "en:nutrition-grade-computed-for-prepared-product"); + } + else { + $product_ref->{"nutrition_grades_tags"} = ["unknown"]; + $product_ref->{nutrition_score_debug} + = "no score for category $category_tag without data for prepared product" . " - "; + add_tag($product_ref, "misc", "en:nutriscore-missing-prepared-nutrition-data"); + $nutrients_available = 0; + } + last; + } + } + # Track the number of key nutrients present my $key_nutrients = 0; @@ -1335,9 +1492,10 @@ sub compute_nutrition_score ($product_ref) { ); $product_ref->{"nutrition_grades_tags"} = ["unknown"]; add_tag($product_ref, "misc", "en:nutrition-not-enough-data-to-compute-nutrition-score"); - $product_ref->{nutrition_score_debug} .= "missing " . $nid . $prepared . " - "; + $product_ref->{nutrition_score_debug} .= "missing " . $nid . $prepared . "_100g - "; add_tag($product_ref, "misc", "en:nutriscore-missing-nutrition-data"); add_tag($product_ref, "misc", "en:nutriscore-missing-nutrition-data-$nid"); + $nutrients_available = 0; } else { $key_nutrients++; @@ -1359,7 +1517,7 @@ sub compute_nutrition_score ($product_ref) { # Remove ending - $product_ref->{nutrition_score_debug} =~ s/ - $//; - # By default we use the "nutriments" hash as a source (specified nutriements), + # By default we use the "nutriments" hash as a source (specified nutrients), # but if we don't have specified nutrients, we can use use the "nutriments_estimated" hash if it exists. # If we have some specified nutrients but are missing required nutrients for the Nutri-Score, # we do not use estimated nutrients, in order to encourage users to complete the nutrition facts @@ -1381,49 +1539,204 @@ sub compute_nutrition_score ($product_ref) { # Delete the warning for missing fiber, as we will get fiber from the estimate delete $product_ref->{nutrition_score_warning_no_fiber}; + + $nutrients_available = 1; } - # If the Nutri-Score is unknown or not applicable, exit the function - if ( - (defined $product_ref->{"nutrition_grades_tags"}) - and ( ($product_ref->{"nutrition_grades_tags"}[0] eq "unknown") - or ($product_ref->{"nutrition_grades_tags"}[0] eq "not-applicable")) - ) + return ($nutrients_available, $prepared, $nutriments_field); +} + +=head2 set_fields_for_current_version_of_nutriscore($product_ref, $current_version, $nutriscore_score, $nutriscore_grade) + +We may compute several versions of the Nutri-Score grade and score. One version is considered "current". +This function sets the product fields for the current version. + +=cut + +sub set_fields_for_current_version_of_nutriscore ($product_ref, $version, $nutriscore_score, $nutriscore_grade) { + + # Record which version is the current version + $product_ref->{nutriscore_version} = $version; + + # Copy the Nutriscore data to nutriscore_data + # to easily see diffs with previous Nutri-Score structure + # (to be deleted once we are sure everything works, + # we will generate the nutriscore_data fields on request + # when asked through the API with an old API version) + # Before 2023-08-29, we did not create a nutriscore_data structure when the Nutri-Score was not computed + # so we do not copy the nutriscore data structure in that case. + if ($product_ref->{nutriscore}{$version}{nutriscore_computed}) { + $product_ref->{nutriscore_data} = dclone($product_ref->{nutriscore}{$version}{data}); + # The grade and score fields are now one level up (not in the data section) + # copy them for backward compatibility + $product_ref->{nutriscore_data}{grade} = $nutriscore_grade; + $product_ref->{nutriscore_data}{score} = $nutriscore_score; + } + + # Copy the resulting values to the main Nutri-Score fields + if (defined $nutriscore_score) { + $product_ref->{nutriscore_score} = $nutriscore_score; + + # Fields used to display the Nutri-Score score inside nutrition facts table + # and to compute averages etc. for categories + $product_ref->{nutriments}{"nutrition-score-fr_100g"} = $nutriscore_score; + $product_ref->{nutriments}{"nutrition-score-fr"} = $nutriscore_score; + + # In order to be able to sort by nutrition score in MongoDB, + # we create an opposite of the nutrition score + # as otherwise, in ascending order on nutriscore_score, we first get products without the nutriscore_score field + # instead we can sort on descending order on nutriscore_score_opposite + $product_ref->{nutriscore_score_opposite} = -$nutriscore_score; + } + $product_ref->{nutriscore_grade} = $nutriscore_grade; + + $product_ref->{"nutrition_grades_tags"} = [$nutriscore_grade]; + $product_ref->{"nutrition_grades"} = $nutriscore_grade; # needed for the /nutrition-grade/unknown query + # (TODO at some point: remove the nutrition_grades field) + + # Gradually rename nutrition_grades_tags to nutriscore_tags + $product_ref->{"nutriscore_tags"} = [$nutriscore_grade]; + + # Legacy field, to be removed from the product and returned by the API on request / for older versions + $product_ref->{"nutrition_grade_fr"} = $nutriscore_grade; + + return; +} + +=head2 set_fields_comparing_nutriscore_versions($product_ref, $version1, $version2) + +When we are migrating from one version of the Nutri-Score to another (e.g. 2021 vs 2023), +we may compute both version for a time. This function sets temporary fields to ease the comparison +of both versions. + +Once the migration is complete, those fields will no longer be computed. + +=cut + +sub set_fields_comparing_nutriscore_versions ($product_ref, $version1, $version2) { + + my $nutriscore1 = $product_ref->{nutriscore}{$version1}{grade}; + my $nutriscore2 = $product_ref->{nutriscore}{$version2}{grade}; + + # Set tags fields for both versions + $product_ref->{"nutriscore_${version1}_tags"} = [$nutriscore1]; + $product_ref->{"nutriscore_${version2}_tags"} = [$nutriscore2]; + + # Compare both versions, only if Nutri-Score has been computed in at least one version + if ( (not $product_ref->{nutriscore}{$version1}{nutriscore_computed}) + or (not $product_ref->{nutriscore}{$version2}{nutriscore_computed})) { return; } - if ($prepared ne '') { - push @{$product_ref->{misc_tags}}, "en:nutrition-grade-computed-for-prepared-product"; + if ($nutriscore1 eq $nutriscore2) { + add_tag($product_ref, "misc", "en:nutriscore-$version1-same-as-$version2"); } + else { + add_tag($product_ref, "misc", "en:nutriscore-$version1-different-from-$version2"); + if ($nutriscore1 lt $nutriscore2) { + add_tag($product_ref, "misc", "en:nutriscore-$version1-better-than-$version2"); + } + else { + add_tag($product_ref, "misc", "en:nutriscore-$version1-worse-than-$version2"); + } + } + + add_tag($product_ref, "misc", "en:nutriscore-$version1-$nutriscore1-$version2-$nutriscore2"); - # Populate the data structure that will be passed to Food::Nutriscore + return; +} - $product_ref->{nutriscore_data} = compute_nutriscore_data($product_ref, $prepared, $nutriments_field); +=head2 compute_nutriscore( $product_ref ) - my ($nutriscore_score, $nutriscore_grade) - = ProductOpener::Nutriscore::compute_nutriscore_score_and_grade($product_ref->{nutriscore_data}); +Determines if we have enough data to compute the Nutri-Score (category + nutrition facts), +and if the Nutri-Score is applicable to the product the category. - $product_ref->{nutriscore_score} = $nutriscore_score; - $product_ref->{nutriscore_grade} = $nutriscore_grade; +Populates the data structure needed to compute the Nutri-Score and computes it. - $product_ref->{nutriments}{"nutrition-score-fr_100g"} = $nutriscore_score; - $product_ref->{nutriments}{"nutrition-score-fr"} = $nutriscore_score; +=cut - $product_ref->{"nutrition_grade_fr"} = $nutriscore_grade; +sub compute_nutriscore ($product_ref, $current_version = "2021") { + + # Initialize values + + $product_ref->{nutrition_score_debug} = ''; + + # Remove any previously existing Nutri-Score related fields + remove_nutriscore_fields($product_ref); - $product_ref->{"nutrition_grades_tags"} = [$product_ref->{"nutrition_grade_fr"}]; - $product_ref->{"nutrition_grades"} - = $product_ref->{"nutrition_grade_fr"}; # needed for the /nutrition-grade/unknown query + my ($category_available, $nutriscore_applicable, $not_applicable_category) + = is_nutriscore_applicable_to_the_product_categories($product_ref); - shift @{$product_ref->{misc_tags}}; - push @{$product_ref->{misc_tags}}, "en:nutriscore-computed"; + my ($nutrients_available, $prepared, $nutriments_field) + = check_availability_of_nutrients_needed_for_nutriscore($product_ref); + + if (not($nutriscore_applicable and $nutrients_available)) { + add_tag($product_ref, "misc", "en:nutriscore-not-computed"); + } + else { + add_tag($product_ref, "misc", "en:nutriscore-computed"); + } + + # 2023/08/10: compute both the 2021 and the 2023 versions of the Nutri-Score + + foreach my $version ("2021", "2023") { + + # Record if we have enough data to compute the Nutri-Score and if the Nutri-Score is applicable to the product categories + deep_set( + $product_ref, + "nutriscore", + $version, + { + "category_available" => $category_available, + "nutriscore_applicable" => $nutriscore_applicable, + "nutrients_available" => $nutrients_available, + "nutriscore_computed" => $nutriscore_applicable * $nutrients_available, + } + ); + + if (defined $not_applicable_category) { + deep_set($product_ref, "nutriscore", $version, "not_applicable_category", $not_applicable_category); + } + + # Populate the data structure that will be passed to Food::Nutriscore + deep_set($product_ref, "nutriscore", $version, "data", + compute_nutriscore_data($product_ref, $prepared, $nutriments_field, $version)); + + # Compute the Nutri-Score + my ($nutriscore_score, $nutriscore_grade); + + if (not $category_available) { + $nutriscore_grade = "unknown"; + } + elsif (not $nutriscore_applicable) { + $nutriscore_grade = "not-applicable"; + } + elsif (not $nutrients_available) { + $nutriscore_grade = "unknown"; + } + else { + ($nutriscore_score, $nutriscore_grade) + = ProductOpener::Nutriscore::compute_nutriscore_score_and_grade( + $product_ref->{nutriscore}{$version}{data}, $version); + } + + # Populate the Nutri-Score fields for the current version + if ($version eq $current_version) { + + set_fields_for_current_version_of_nutriscore($product_ref, $current_version, $nutriscore_score, + $nutriscore_grade); + } + + $product_ref->{nutriscore}{$version}{grade} = $nutriscore_grade; + if (defined $nutriscore_score) { + $product_ref->{nutriscore}{$version}{score} = $nutriscore_score; + } + } - # In order to be able to sort by nutrition score in MongoDB, - # we create an opposite of the nutrition score - # as otherwise, in ascending order on nutriscore_score, we first get products without the nutriscore_score field - # instead we can sort on descending order on nutriscore_score_opposite - $product_ref->{nutriscore_score_opposite} = -$nutriscore_score; + # 2023/08/17: as we are migrating from one version of the Nutri-Score to another, we set temporary fields + # to compare both versions. + set_fields_comparing_nutriscore_versions($product_ref, "2021", "2023"); return; } diff --git a/lib/ProductOpener/Import.pm b/lib/ProductOpener/Import.pm index acd7f5c2c09cf..bae1e2e69e07e 100644 --- a/lib/ProductOpener/Import.pm +++ b/lib/ProductOpener/Import.pm @@ -3027,7 +3027,7 @@ sub import_products_categories_from_public_database ($args_ref) { $log->debug("Food::special_process_product") if $log->is_debug(); ProductOpener::Food::special_process_product($product_ref); } - compute_nutrition_score($product_ref); + compute_nutriscore($product_ref); compute_nova_group($product_ref); compute_nutrient_levels($product_ref); compute_unknown_nutrients($product_ref); diff --git a/lib/ProductOpener/Nutriscore.pm b/lib/ProductOpener/Nutriscore.pm index 126f711eff5a3..1aacb8b1d47d9 100644 --- a/lib/ProductOpener/Nutriscore.pm +++ b/lib/ProductOpener/Nutriscore.pm @@ -44,7 +44,8 @@ of a food product. is_beverage => 1, is_water => 0, is_cheese => 0, - is_fat => 0, + is_fat => 1, # for 2021 version + is_fat_nuts_seed => 1, # for 2023 version } my ($nutriscore_score, $nutriscore_grade) = compute_nutriscore_score_and_grade( @@ -83,6 +84,14 @@ BEGIN { &get_value_with_one_less_negative_point &get_value_with_one_more_positive_point + %points_thresholds_2023 + + &compute_nutriscore_score_and_grade_2023 + &compute_nutriscore_grade_2023 + + &get_value_with_one_less_negative_point_2023 + &get_value_with_one_more_positive_point_2023 + ); # symbols to export on request %EXPORT_TAGS = (all => [@EXPORT_OK]); } @@ -91,14 +100,48 @@ use vars @EXPORT_OK; =head1 FUNCTIONS -=head2 compute_nutriscore_score_and_grade( $nutriscore_data_ref ) +Note: a significantly different Nutri-Score algorithm has been published in 2022 and 2023 +(respectively for foods and beverages). + +The functions related to the previous algorithm will be suffixed with _2021, +and functions for the new algorithm will be suffixed with _2023. + +We will keep both algorithms for a transition period, and we will be able to remove the +2021 algorithm and related functions after. + +=cut + +sub compute_nutriscore_score_and_grade ($nutriscore_data_ref, $version = "2021") { + + if ($version eq "2023") { + return compute_nutriscore_score_and_grade_2023($nutriscore_data_ref); + } + return compute_nutriscore_score_and_grade_2021($nutriscore_data_ref); +} + +# methods returning the 2021 version for now, to ease switch, later on. +sub get_value_with_one_less_negative_point ($nutriscore_data_ref, $nutrient) { + return get_value_with_one_less_negative_point_2021($nutriscore_data_ref, $nutrient); +} + +sub get_value_with_one_more_positive_point ($nutriscore_data_ref, $nutrient) { + return get_value_with_one_more_positive_point_2021($nutriscore_data_ref, $nutrient); +} + +sub compute_nutriscore_grade ($nutrition_score, $is_beverage, $is_water) { + return compute_nutriscore_grade_2021($nutrition_score, $is_beverage, $is_water); +} + +# 2021 algorithm -C computes the Nutri-Score score and grade +=head2 compute_nutriscore_score_and_grade_2021( $nutriscore_data_ref ) + +Computes the Nutri-Score score and grade of a food product, and also returns the details of the points for each nutrient. =head3 Arguments: $nutriscore_data_ref -1 hash references need to be passed as arguments. It is used for both input and output: +Hash reference used for both input and output: =head4 Input keys: data to compute Nutri-Score @@ -129,8 +172,6 @@ Returned values: - [nutrient]_points -> points for each nutrient - negative_points -> sum of unfavorable nutrients points - positive_points -> sum of favorable nutrients points -- score -> nutrition score -- grade -> Nutri-Score grade (A ti E The nutrients that are counted for the negative and positive points depend on the product type (if it is a beverage, cheese or fat) and on the values for some of the nutrients. @@ -146,27 +187,24 @@ The letter grade depends on the score and on whether the product is a beverage, =cut -sub compute_nutriscore_score_and_grade ($nutriscore_data_ref) { +sub compute_nutriscore_score_and_grade_2021 ($nutriscore_data_ref) { # We will pass a %point structure to get the details of the computation # so that it can be returned my %points = (); - my $nutrition_score = compute_nutriscore_score($nutriscore_data_ref); + my $nutrition_score = compute_nutriscore_score_2021($nutriscore_data_ref); - my $nutrition_grade = compute_nutriscore_grade( + my $nutrition_grade = compute_nutriscore_grade_2021( $nutrition_score, $nutriscore_data_ref->{is_beverage}, $nutriscore_data_ref->{is_water} ); - $nutriscore_data_ref->{score} = $nutrition_score; - $nutriscore_data_ref->{grade} = $nutrition_grade; - return ($nutrition_score, $nutrition_grade); } -%points_thresholds = ( +my %points_thresholds_2021 = ( # negative points @@ -186,7 +224,7 @@ sub compute_nutriscore_score_and_grade ($nutriscore_data_ref) { proteins => [1.6, 3.2, 4.8, 6.4, 8.0] # g / 100g ); -=head2 get_value_with_one_less_negative_point( $nutriscore_data_ref, $nutrient ) +=head2 get_value_with_one_less_negative_point_2021 ( $nutriscore_data_ref, $nutrient ) For a given Nutri-Score nutrient value, return the highest smaller value that would result in less negative points. e.g. for a sugars value of 15 (which gives 3 points), return 13.5 (which gives 2 points). @@ -197,19 +235,19 @@ Return undef is the input nutrient value already gives the minimum amount of poi =cut -sub get_value_with_one_less_negative_point ($nutriscore_data_ref, $nutrient) { +sub get_value_with_one_less_negative_point_2021 ($nutriscore_data_ref, $nutrient) { my $nutrient_threshold_id = $nutrient; if ( (defined $nutriscore_data_ref->{is_beverage}) and ($nutriscore_data_ref->{is_beverage}) - and (defined $points_thresholds{$nutrient_threshold_id . "_beverages"})) + and (defined $points_thresholds_2021{$nutrient_threshold_id . "_beverages"})) { $nutrient_threshold_id .= "_beverages"; } my $lower_threshold; - foreach my $threshold (@{$points_thresholds{$nutrient_threshold_id}}) { + foreach my $threshold (@{$points_thresholds_2021{$nutrient_threshold_id}}) { # The saturated fat ratio table uses the greater or equal sign instead of greater if ( (($nutrient eq "saturated_fat_ratio") and ($nutriscore_data_ref->{$nutrient . "_value"} >= $threshold)) or (($nutrient ne "saturated_fat_ratio") and ($nutriscore_data_ref->{$nutrient . "_value"} > $threshold))) @@ -221,7 +259,7 @@ sub get_value_with_one_less_negative_point ($nutriscore_data_ref, $nutrient) { return $lower_threshold; } -=head2 get_value_with_one_more_positive_point( $nutriscore_data_ref, $nutrient ) +=head2 get_value_with_one_more_positive_point_2021 ( $nutriscore_data_ref, $nutrient ) For a given Nutri-Score nutrient value, return the smallest higher value that would result in more positive points. e.g. for a proteins value of 2.0 (which gives 1 point), return 3.3 (which gives 2 points) @@ -232,19 +270,19 @@ Return undef is the input nutrient value already gives the maximum amount of poi =cut -sub get_value_with_one_more_positive_point ($nutriscore_data_ref, $nutrient) { +sub get_value_with_one_more_positive_point_2021 ($nutriscore_data_ref, $nutrient) { my $nutrient_threshold_id = $nutrient; if ( (defined $nutriscore_data_ref->{is_beverage}) and ($nutriscore_data_ref->{is_beverage}) - and (defined $points_thresholds{$nutrient_threshold_id . "_beverages"})) + and (defined $points_thresholds_2021{$nutrient_threshold_id . "_beverages"})) { $nutrient_threshold_id .= "_beverages"; } my $higher_threshold; - foreach my $threshold (@{$points_thresholds{$nutrient_threshold_id}}) { + foreach my $threshold (@{$points_thresholds_2021{$nutrient_threshold_id}}) { if ($nutriscore_data_ref->{$nutrient . "_value"} < $threshold) { $higher_threshold = $threshold; last; @@ -267,16 +305,33 @@ sub get_value_with_one_more_positive_point ($nutriscore_data_ref, $nutrient) { return $return_value; } -sub compute_nutriscore_score ($nutriscore_data_ref) { +sub compute_nutriscore_score_2021 ($nutriscore_data_ref) { + + # If the product is in fats and oils category, + # the saturated fat points are replaced by the saturated fat / fat ratio points + + my $saturated_fat = "saturated_fat"; + if ($nutriscore_data_ref->{is_fat}) { + $saturated_fat = "saturated_fat_ratio"; + } # The values must be rounded with one more digit than the thresolds. # Undefined values are counted as 0 (it can be the case in particular for waters that have different nutrients listed) + # Note (2023/08/08): there used to be a rounding rule (e.g. in the Nutri-Score specification UPDATED 12/05/2020): + # "Points are assigned for a given nutrient based on the content of the nutrient in question in the food, with a + # rounded value corresponding to one additional number with regard to the point attribution threshold." + # + # This rule has been changed (e.g. in spec UPDATED 27/09/2022): + # Points are assigned according to the values indicated on the mandatory nutritional declaration. + # To determine number of decimals needed, we recommend the use of the European guidance document + # with regards to the settings of tolerances for nutrient values for labels. For optional nutrients, in accordance + # with Article 30-2 of the INCO regulation 1169/2011, such as fibre, rounding guidelines from the previous + # document are also recommended. + # Round with 1 digit after the comma for energy, saturated fat, saturated fat ratio, sodium and fruits - foreach my $nutrient ( - qw(energy saturated_fat saturated_fat_ratio sodium fruits_vegetables_nuts_colza_walnut_olive_oils)) - { + foreach my $nutrient ("energy", $saturated_fat, "sodium", "fruits_vegetables_nuts_colza_walnut_olive_oils") { if (defined $nutriscore_data_ref->{$nutrient}) { $nutriscore_data_ref->{$nutrient . "_value"} = int($nutriscore_data_ref->{$nutrient} * 10 + 0.5) / 10; } @@ -287,7 +342,7 @@ sub compute_nutriscore_score ($nutriscore_data_ref) { # Round with 2 digits for sugars, fiber and proteins - foreach my $nutrient (qw(sugars fiber proteins)) { + foreach my $nutrient ("sugars", "fiber", "proteins") { if (defined $nutriscore_data_ref->{$nutrient}) { $nutriscore_data_ref->{$nutrient . "_value"} = int($nutriscore_data_ref->{$nutrient} * 100 + 0.5) / 100; } @@ -306,22 +361,22 @@ sub compute_nutriscore_score ($nutriscore_data_ref) { # Compute the negative and positive points - foreach my $nutrient ( - qw(energy sugars saturated_fat saturated_fat_ratio sodium fruits_vegetables_nuts_colza_walnut_olive_oils fiber proteins) - ) + foreach + my $nutrient ("energy", "sugars", $saturated_fat, "sodium", "fruits_vegetables_nuts_colza_walnut_olive_oils", + "fiber", "proteins") { my $nutrient_threshold_id = $nutrient; if ( (defined $nutriscore_data_ref->{is_beverage}) and ($nutriscore_data_ref->{is_beverage}) - and (defined $points_thresholds{$nutrient_threshold_id . "_beverages"})) + and (defined $points_thresholds_2021{$nutrient_threshold_id . "_beverages"})) { $nutrient_threshold_id .= "_beverages"; } $nutriscore_data_ref->{$nutrient . "_points"} = 0; - foreach my $threshold (@{$points_thresholds{$nutrient_threshold_id}}) { + foreach my $threshold (@{$points_thresholds_2021{$nutrient_threshold_id}}) { # The saturated fat ratio table uses the greater or equal sign instead of greater if ( ( @@ -339,16 +394,8 @@ sub compute_nutriscore_score ($nutriscore_data_ref) { # Negative points - # If the product is an added fat (oil, butter etc.) the saturated fat points are replaced - # by the saturated fat / fat ratio points - - my $fat = "saturated_fat"; - if ((defined $nutriscore_data_ref->{is_fat}) and ($nutriscore_data_ref->{is_fat})) { - $fat = "saturated_fat_ratio"; - } - $nutriscore_data_ref->{negative_points} = 0; - foreach my $nutrient ("energy", "sugars", $fat, "sodium") { + foreach my $nutrient ("energy", "sugars", $saturated_fat, "sodium") { $nutriscore_data_ref->{negative_points} += $nutriscore_data_ref->{$nutrient . "_points"}; } @@ -383,7 +430,7 @@ sub compute_nutriscore_score ($nutriscore_data_ref) { return $score; } -sub compute_nutriscore_grade ($nutrition_score, $is_beverage, $is_water) { +sub compute_nutriscore_grade_2021 ($nutrition_score, $is_beverage, $is_water) { my $grade = ""; @@ -430,5 +477,366 @@ sub compute_nutriscore_grade ($nutrition_score, $is_beverage, $is_water) { return $grade; } +# 2022 / 2023 algorithm + +=head2 compute_nutriscore_score_and_grade_2023( $nutriscore_data_ref ) + +Computes the Nutri-Score score and grade +of a food product, and also returns the details of the points for each nutrient. + +=head3 Arguments: $nutriscore_data_ref + +Hash reference used for both input and output: + +=head4 Input keys: data to compute Nutri-Score + +The hash must contain values for the following keys: + +- energy -> energy in kJ / 100g or 100ml +- sugars -> sugars in g / 100g or 100ml +- saturated_fat -> saturated fats in g / 100g or 100ml +- saturated_fat_ratio -> saturated fat divided by fat * 100 (in %) +- sodium -> sodium in mg / 100g or 100ml (if sodium is computed from salt, it needs to use a sodium = salt / 2.5 conversion factor +- fruits_vegetables_nuts_colza_walnut_olive_oils -> % of fruits, vegetables, nuts, and colza / walnut / olive oils +- fiber -> fiber in g / 100g or 100ml +- proteins -> proteins in g / 100g or 100ml + +The values will be rounded according to the Nutri-Score rules, they do not need to be rounded before being passed as arguments. + +If the product is a beverage, water, cheese, or fat, it must contain a positive value for the corresponding keys: +- is_beverage +- is_water +- is_cheese +- is_fat_oil_nuts_seeds + +=head4 Output keys: details of the Nutri-Score computation + +Returned values: + +- [nutrient]_value -> rounded values for each nutrient according to the Nutri-Score rules +- [nutrient]_points -> points for each nutrient +- negative_nutrients -> list of nutrients that are counted in negative points +- negative_points -> sum of unfavorable nutrients points +- positive_nutrients -> list of nutrients that are counted in positive points +- positive_points -> sum of favorable nutrients points +- count_proteins -> indicates if proteins are counted in the positive points +- count_proteins_reason -> indicates why proteins are counted + +The nutrients that are counted for the negative and positive points depend on the product type +(if it is a beverage, cheese or fat) and on the values for some of the nutrients. + +=head3 Return values + +The function returns a list of 2 values: + +- Nutri-Score score from -15 to 40 +- Corresponding Nutri-Score letter grade from a to e (in lowercase) + +The letter grade depends on the score and on whether the product is a beverage, or is a water. + +=cut + +sub compute_nutriscore_score_and_grade_2023 ($nutriscore_data_ref) { + + # We will pass a %point structure to get the details of the computation + # so that it can be returned + my %points = (); + + my $nutrition_score = compute_nutriscore_score_2023($nutriscore_data_ref); + + my $nutrition_grade = compute_nutriscore_grade_2023( + $nutrition_score, + $nutriscore_data_ref->{is_beverage}, + $nutriscore_data_ref->{is_water} + ); + + return ($nutrition_score, $nutrition_grade); +} + +my %points_thresholds_2023 = ( + + # negative points + + energy => [335, 670, 1005, 1340, 1675, 2010, 2345, 2680, 3015, 3350], # kJ / 100g + energy_beverages => [30, 90, 150, 210, 240, 270, 300, 330, 360, 390], # kJ /100g or 100ml + sugars => [3.4, 6.8, 10, 14, 17, 20, 24, 27, 31, 34, 37, 41, 44, 48, 51], # g / 100g + sugars_beverages => [0.5, 2, 3.5, 5, 6, 7, 8, 9, 10, 11], # g / 100g or 100ml + saturated_fat => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], # g / 100g + salt => [0.2, 0.4, 0.6, 0.8, 1, 1.2, 1.4, 1.6, 1.8, 2, 2.2, 2.4, 2.6, 2.8, 3, 3.2, 3.2, 3.4, 3.6, 3.8, 4] + , # g / 100g + + # for fats + energy_from_saturated_fat => [120, 240, 360, 480, 600, 720, 840, 960, 1080, 1200], # g / 100g + saturated_fat_ratio => [10, 16, 22, 28, 34, 40, 46, 52, 58, 64], # % + + # positive points + fruits_vegetables_legumes => [40, 60, 80, 80, 80], # % + fruits_vegetables_legumes_beverages => [40, 40, 60, 60, 80, 80], + fiber => [3.0, 4.1, 5.2, 6.3, 7.4], # g / 100g - AOAC method + proteins => [2.4, 4.8, 7.2, 9.6, 12, 14, 17], # g / 100g + proteins_beverages => [1.2, 1.5, 1.8, 2.1, 2.4, 2.7, 3.0], # g / 100g +); + +=head2 get_value_with_one_less_negative_point_2023( $nutriscore_data_ref, $nutrient ) + +For a given Nutri-Score nutrient value, return the highest smaller value that would result in less negative points. +e.g. for a sugars value of 15 (which gives 3 points), return 13.5 (which gives 2 points). + +The value corresponds to the highest smaller threshold. + +Return undef if the input nutrient value already gives the minimum amount of points (0). + +=cut + +sub get_value_with_one_less_negative_point_2023 ($nutriscore_data_ref, $nutrient) { + + my $nutrient_threshold_id = $nutrient; + if ( (defined $nutriscore_data_ref->{is_beverage}) + and ($nutriscore_data_ref->{is_beverage}) + and (defined $points_thresholds_2023{$nutrient_threshold_id . "_beverages"})) + { + $nutrient_threshold_id .= "_beverages"; + } + + my $lower_threshold; + + foreach my $threshold (@{$points_thresholds_2023{$nutrient_threshold_id}}) { + # The saturated fat ratio table uses the greater or equal sign instead of greater + if ( (($nutrient eq "saturated_fat_ratio") and ($nutriscore_data_ref->{$nutrient . "_value"} >= $threshold)) + or (($nutrient ne "saturated_fat_ratio") and ($nutriscore_data_ref->{$nutrient . "_value"} > $threshold))) + { + $lower_threshold = $threshold; + } + } + + return $lower_threshold; +} + +=head2 get_value_with_one_more_positive_point_2023( $nutriscore_data_ref, $nutrient ) + +For a given Nutri-Score nutrient value, return the smallest higher value that would result in more positive points. +e.g. for a proteins value of 2.0 (which gives 1 point), return 3.3 (which gives 2 points) + +The value corresponds to the smallest higher threshold + 1 increment so that it strictly greater than the threshold. + +Return undef if the input nutrient value already gives the maximum amount of points. + +=cut + +sub get_value_with_one_more_positive_point_2023 ($nutriscore_data_ref, $nutrient) { + + my $nutrient_threshold_id = $nutrient; + if ( (defined $nutriscore_data_ref->{is_beverage}) + and ($nutriscore_data_ref->{is_beverage}) + and (defined $points_thresholds_2023{$nutrient_threshold_id . "_beverages"})) + { + $nutrient_threshold_id .= "_beverages"; + } + + my $higher_threshold; + + foreach my $threshold (@{$points_thresholds_2023{$nutrient_threshold_id}}) { + if ($nutriscore_data_ref->{$nutrient . "_value"} < $threshold) { + $higher_threshold = $threshold; + last; + } + } + + # The return value needs to be strictly greater than the threshold + + my $return_value = $higher_threshold; + + if ($return_value) { + if ($nutrient eq "fruits_vegetables_nuts_colza_walnut_olive_oils") { + $return_value += 1; + } + else { + $return_value += 0.1; + } + } + + return $return_value; +} + +sub compute_nutriscore_score_2023 ($nutriscore_data_ref) { + + # The values must be rounded with one more digit than the thresholds. + # Undefined values are counted as 0 (it can be the case in particular for waters that have different nutrients listed) + + # The Nutri-Score FAQ has been updated (UPDATED 27/09/2022 or earleier), + # and there is now no rounding of nutrient values + + # Points are assigned according to the values indicated on the mandatory nutritional declaration. + # To determine number of decimals needed, we recommend the use of the European guidance document + # with regards to the settings of tolerances for nutrient values for labels. For optional nutrients, in accordance + # with Article 30-2 of the INCO regulation 1169/2011, such as fibre, rounding guidelines from the previous + # document are also recommended. + + # Compute the negative and positive points + + # If the product is in fats, oils, nuts and seeds category, + # the energy points are replaced by the energy from saturates points, and + # the saturated fat points are replaced by the saturated fat / fat ratio points + + my $energy = "energy"; + my $saturated_fat = "saturated_fat"; + if ($nutriscore_data_ref->{is_fat_oil_nuts_seeds}) { + $saturated_fat = "saturated_fat_ratio"; + $energy = "energy_from_saturated_fat"; + } + + foreach my $nutrient ($energy, "sugars", $saturated_fat, "salt", "fruits_vegetables_legumes", "fiber", "proteins") { + next if not defined $nutriscore_data_ref->{$nutrient}; + + my $nutrient_threshold_id = $nutrient; + if ( (defined $nutriscore_data_ref->{is_beverage}) + and ($nutriscore_data_ref->{is_beverage}) + and (defined $points_thresholds_2023{$nutrient_threshold_id . "_beverages"})) + { + $nutrient_threshold_id .= "_beverages"; + } + + $nutriscore_data_ref->{$nutrient . "_points"} = 0; + + foreach my $threshold (@{$points_thresholds_2023{$nutrient_threshold_id}}) { + # The saturated fat ratio table uses the greater or equal sign instead of greater + if ( (($nutrient eq "saturated_fat_ratio") and ($nutriscore_data_ref->{$nutrient} >= $threshold)) + or (($nutrient ne "saturated_fat_ratio") and ($nutriscore_data_ref->{$nutrient} > $threshold))) + { + $nutriscore_data_ref->{$nutrient . "_points"}++; + } + } + } + + # For red meat products, the number of maximum protein points is set at 2 points + # Red meat products qualifying for this specific rule are products from beef, veal, swine and lamb, + # though they include also game/venison, horse, donkey, goat, camel and kangaroo. + + if (($nutriscore_data_ref->{is_read_meat_products}) and ($nutriscore_data_ref->{proteins_points} > 2)) { + $nutriscore_data_ref->{proteins_points} = 2; + } + + # Beverages with non-nutritive sweeteners have 4 extra negative points + if ($nutriscore_data_ref->{is_beverage}) { + if ($nutriscore_data_ref->{has_sweeteners}) { + $nutriscore_data_ref->{"sweeteners_points"} = 4; + } + else { + $nutriscore_data_ref->{"sweeteners_points"} = 0; + } + } + + # Negative points + + $nutriscore_data_ref->{negative_nutrients} = [$energy, "sugars", $saturated_fat, "salt", "sweeteners"]; + $nutriscore_data_ref->{negative_points} = 0; + foreach my $nutrient (@{$nutriscore_data_ref->{negative_nutrients}}) { + $nutriscore_data_ref->{negative_points} += ($nutriscore_data_ref->{$nutrient . "_points"} || 0); + } + + # Positive points + + $nutriscore_data_ref->{positive_points} = 0; + $nutriscore_data_ref->{positive_nutrients} = ["fruits_vegetables_legumes", "fiber"]; + + # Positive points for proteins are counted in the following 3 cases: + # - the product is a beverage + # - the product is not in the fats, oils, nuts and seeds category + # and the negative points are less than 11 or the product is a cheese + # - the product is in the fats, oils, nuts and seeds category + # and the negative points are less than 7 + + $nutriscore_data_ref->{count_proteins} = 0; + if ($nutriscore_data_ref->{is_beverage}) { + $nutriscore_data_ref->{count_proteins} = 1; + $nutriscore_data_ref->{count_proteins_reason} = "beverage"; + } + elsif ($nutriscore_data_ref->{is_cheese}) { + $nutriscore_data_ref->{count_proteins} = 1; + $nutriscore_data_ref->{count_proteins_reason} = "cheese"; + } + else { + if ($nutriscore_data_ref->{is_fat_oil_nuts_seeds}) { + if ($nutriscore_data_ref->{negative_points} < 7) { + $nutriscore_data_ref->{count_proteins} = 1; + $nutriscore_data_ref->{count_proteins_reason} = "negative_points_less_than_7"; + } + else { + $nutriscore_data_ref->{count_proteins_reason} = "negative_points_more_than_7"; + } + } + else { + if ($nutriscore_data_ref->{negative_points} < 11) { + $nutriscore_data_ref->{count_proteins} = 1; + $nutriscore_data_ref->{count_proteins_reason} = "negative_points_less_than_11"; + } + else { + $nutriscore_data_ref->{count_proteins_reason} = "negative_points_more_than_11"; + } + } + } + + if ($nutriscore_data_ref->{count_proteins}) { + push @{$nutriscore_data_ref->{positive_nutrients}}, "proteins"; + } + + foreach my $nutrient (@{$nutriscore_data_ref->{positive_nutrients}}) { + $nutriscore_data_ref->{positive_points} += ($nutriscore_data_ref->{$nutrient . "_points"} || 0); + } + + my $score = $nutriscore_data_ref->{negative_points} - $nutriscore_data_ref->{positive_points}; + + return $score; +} + +sub compute_nutriscore_grade_2023 ($nutrition_score, $is_beverage, $is_water) { + + my $grade = ""; + + if (not defined $nutrition_score) { + return ''; + } + + if ($is_beverage) { + + if ($is_water) { + $grade = 'a'; + } + elsif ($nutrition_score <= 2) { + $grade = 'b'; + } + elsif ($nutrition_score <= 6) { + $grade = 'c'; + } + elsif ($nutrition_score <= 9) { + $grade = 'd'; + } + else { + $grade = 'e'; + } + } + else { + + if ($nutrition_score <= -6) { + $grade = 'a'; + } + elsif ($nutrition_score <= 2) { + $grade = 'b'; + } + elsif ($nutrition_score <= 10) { + $grade = 'c'; + } + elsif ($nutrition_score <= 18) { + $grade = 'd'; + } + else { + $grade = 'e'; + } + } + return $grade; +} + +%points_thresholds = %points_thresholds_2021; + 1; diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm index 55489bbec5238..e937653f1f42e 100644 --- a/lib/ProductOpener/Products.pm +++ b/lib/ProductOpener/Products.pm @@ -3529,6 +3529,9 @@ sub analyze_and_enrich_product_data ($product_ref, $response_ref) { $log->debug("analyze_and_enrich_product_data - start") if $log->is_debug(); + # Initialiaze the misc_tags, they will be populated by functions called by this function + $product_ref->{misc_tags} = []; + if ( (defined $product_ref->{nutriments}{"carbon-footprint"}) and ($product_ref->{nutriments}{"carbon-footprint"} ne '')) { @@ -3577,7 +3580,7 @@ sub analyze_and_enrich_product_data ($product_ref, $response_ref) { compute_estimated_nutrients($product_ref); - compute_nutrition_score($product_ref); + compute_nutriscore($product_ref); compute_nova_group($product_ref); diff --git a/po/tags/en.po b/po/tags/en.po index 710610a828da1..76624460a2b88 100644 --- a/po/tags/en.po +++ b/po/tags/en.po @@ -590,3 +590,32 @@ msgctxt "packaging_recycling:singular" msgid "packaging-recycling" msgstr "packaging-recycling" +# Please do not translate nutri-score +msgctxt "nutriscore:plural" +msgid "nutri-score" +msgstr "nutri-score" + +# Please do not translate nutri-score +msgctxt "nutriscore:singular" +msgid "nutri-score" +msgstr "nutri-score" + +# Please do not translate nutri-score +msgctxt "nutriscore_2021:plural" +msgid "nutri-score-2021" +msgstr "nutri-score-2021" + +# Please do not translate nutri-score +msgctxt "nutriscore_2021:singular" +msgid "nutri-score-2021" +msgstr "nutri-score-2021" + +# Please do not translate nutri-score +msgctxt "nutriscore_2023:plural" +msgid "nutri-score-2023" +msgstr "nutri-score-2023" + +# Please do not translate nutri-score +msgctxt "nutriscore_2023:singular" +msgid "nutri-score-2023" +msgstr "nutri-score-2023" \ No newline at end of file diff --git a/po/tags/tags.pot b/po/tags/tags.pot index 1d9fc990a8c8c..e40408580d275 100644 --- a/po/tags/tags.pot +++ b/po/tags/tags.pot @@ -694,3 +694,32 @@ msgctxt "packaging_recycling:singular" msgid "packaging-recycling" msgstr "packaging-recycling" +# Please do not translate nutri-score +msgctxt "nutriscore:plural" +msgid "nutri-score" +msgstr "nutri-score" + +# Please do not translate nutri-score +msgctxt "nutriscore:singular" +msgid "nutri-score" +msgstr "nutri-score" + +# Please do not translate nutri-score +msgctxt "nutriscore_2021:plural" +msgid "nutri-score-2021" +msgstr "nutri-score-2021" + +# Please do not translate nutri-score +msgctxt "nutriscore_2021:singular" +msgid "nutri-score-2021" +msgstr "nutri-score-2021" + +# Please do not translate nutri-score +msgctxt "nutriscore_2023:plural" +msgid "nutri-score-2023" +msgstr "nutri-score-2023" + +# Please do not translate nutri-score +msgctxt "nutriscore_2023:singular" +msgid "nutri-score-2023" +msgstr "nutri-score-2023" \ No newline at end of file diff --git a/scripts/aggregate_ingredients.pl b/scripts/aggregate_ingredients.pl index 3296db8760b87..f8bc88b935038 100755 --- a/scripts/aggregate_ingredients.pl +++ b/scripts/aggregate_ingredients.pl @@ -54,26 +54,6 @@ use Encode; use JSON::PP; -#use Getopt::Long; -# -#my @fields_to_update = (); -#my $key; -#my $index = ''; -#my $pretend = ''; -#my $process_ingredients = ''; -#my $compute_nutrition_score = ''; -#my $check_quality = ''; -# -#GetOptions ("key=s" => \$key, # string -# "fields=s" => \@fields_to_update, -# "index" => \$index, -# "pretend" => \$pretend, -# "process-ingredients" => \$process_ingredients, -# "compute-nutrition-score" => \$compute_nutrition_score, -# "check-quality" => \$check_quality, -# ) -# or die("Error in command line arguments:\n$\nusage"); - my $query_ref = {}; my $cursor = get_products_collection()->query($query_ref)->fields({code => 1}); diff --git a/scripts/extract_individual_ingredients.pl b/scripts/extract_individual_ingredients.pl index 97634799d9a2f..51c4cc8b70ae5 100755 --- a/scripts/extract_individual_ingredients.pl +++ b/scripts/extract_individual_ingredients.pl @@ -54,26 +54,6 @@ use Encode; use JSON::PP; -#use Getopt::Long; -# -#my @fields_to_update = (); -#my $key; -#my $index = ''; -#my $pretend = ''; -#my $process_ingredients = ''; -#my $compute_nutrition_score = ''; -#my $check_quality = ''; -# -#GetOptions ("key=s" => \$key, # string -# "fields=s" => \@fields_to_update, -# "index" => \$index, -# "pretend" => \$pretend, -# "process-ingredients" => \$process_ingredients, -# "compute-nutrition-score" => \$compute_nutrition_score, -# "check-quality" => \$check_quality, -# ) -# or die("Error in command line arguments:\n$\nusage"); - my $query_ref = {}; my $products_collection = get_products_collection(); diff --git a/scripts/import_fleurymichon.pl b/scripts/import_fleurymichon.pl index fbc398ecf47da..d5eec7a5d2298 100755 --- a/scripts/import_fleurymichon.pl +++ b/scripts/import_fleurymichon.pl @@ -877,70 +877,10 @@ } #exit; - # Process the fields - - # Food category rules for sweeetened/sugared beverages - # French PNNS groups from categories - - if ($server_domain =~ /openfoodfacts/) { - ProductOpener::Food::special_process_product($product_ref); - } - - if ( (defined $product_ref->{nutriments}{"carbon-footprint"}) - and ($product_ref->{nutriments}{"carbon-footprint"} ne '')) - { - push @{$product_ref->{"labels_hierarchy"}}, "en:carbon-footprint"; - push @{$product_ref->{"labels_tags"}}, "en:carbon-footprint"; - } - - if ((defined $product_ref->{nutriments}{"glycemic-index"}) and ($product_ref->{nutriments}{"glycemic-index"} ne '')) - { - push @{$product_ref->{"labels_hierarchy"}}, "en:glycemic-index"; - push @{$product_ref->{"labels_tags"}}, "en:glycemic-index"; - } - - # Language and language code / subsite - - if (defined $product_ref->{lang}) { - $product_ref->{lc} = $product_ref->{lang}; - } - - if (not defined $lang_lc{$product_ref->{lc}}) { - $product_ref->{lc} = 'xx'; - } - - # For fields that can have different values in different languages, copy the main language value to the non suffixed field - - foreach my $field (keys %language_fields) { - if ($field !~ /_image/) { - if (defined $product_ref->{$field . "_$product_ref->{lc}"}) { - $product_ref->{$field} = $product_ref->{$field . "_$product_ref->{lc}"}; - } - } - } - - # Ingredients classes - extract_ingredients_from_text($product_ref); - extract_ingredients_classes_from_text($product_ref); - - compute_languages($product_ref); # need languages for allergens detection - detect_allergens_from_text($product_ref); + $User_id = $editor_user_id; - #"sources": [ - #{ - #"id", "usda-ndb", - #"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available) - #"import_t", "423423" (timestamp of import date) - #"fields" : ["product_name","ingredients","nutrients"] - #"images" : [ "1", "2", "3" ] (images ids) - #}, - #{ - #"id", "usda-ndb", - #"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available) - #"import_t", "523423" (timestamp of import date) - #"fields" : ["ingredients","nutrients"] - #"images" : [ "4", "5", "6" ] (images ids) - #}, + my $response_ref = {}; + analyze_and_enrich_product_data($product_ref, $response_ref); if (not defined $product_ref->{sources}) { $product_ref->{sources} = []; @@ -957,25 +897,8 @@ images => \@images_ids, }; - $User_id = $editor_user_id; - if (not $testing) { - fix_salt_equivalent($product_ref); - - compute_serving_size_data($product_ref); - - compute_nutrition_score($product_ref); - - compute_nutrient_levels($product_ref); - - compute_unknown_nutrients($product_ref); - - #print STDERR "Storing product code $code\n"; - # use Data::Dumper; - #print STDERR Dumper($product_ref); - #exit; - store_product($User_id, $product_ref, "Editing product (import_fleurymichon_ch.pl bulk import) - " . $comment); push @edited, $code; diff --git a/scripts/import_systemeu.pl b/scripts/import_systemeu.pl index eba2dcd860808..41861da9982ff 100755 --- a/scripts/import_systemeu.pl +++ b/scripts/import_systemeu.pl @@ -1549,115 +1549,10 @@ # Process the fields - # Food category rules for sweeetened/sugared beverages - # French PNNS groups from categories - - if ($server_domain =~ /openfoodfacts/) { - ProductOpener::Food::special_process_product($product_ref); - } - - if ( (defined $product_ref->{nutriments}{"carbon-footprint"}) - and ($product_ref->{nutriments}{"carbon-footprint"} ne '')) - { - push @{$product_ref->{"labels_hierarchy"}}, "en:carbon-footprint"; - push @{$product_ref->{"labels_tags"}}, "en:carbon-footprint"; - } - - if ((defined $product_ref->{nutriments}{"glycemic-index"}) and ($product_ref->{nutriments}{"glycemic-index"} ne '')) - { - push @{$product_ref->{"labels_hierarchy"}}, "en:glycemic-index"; - push @{$product_ref->{"labels_tags"}}, "en:glycemic-index"; - } - - # Language and language code / subsite - - if (defined $product_ref->{lang}) { - $product_ref->{lc} = $product_ref->{lang}; - } - - # For fields that can have different values in different languages, copy the main language value to the non suffixed field - - foreach my $field (keys %language_fields) { - if ($field !~ /_image/) { - if (defined $product_ref->{$field . "_$product_ref->{lc}"}) { - $product_ref->{$field} = $product_ref->{$field . "_$product_ref->{lc}"}; - } - } - } - - if ($testing_allergens) { - - $product_ref->{allergens} = ""; - $product_ref->{traces} = ""; - } - - if ($server_domain =~ /openfoodfacts/) { - ProductOpener::Food::special_process_product($product_ref); - } - - if (($testing_allergens) or (not $testing)) { - # Ingredients classes - print STDERR "computing allergens etc.\n"; - - extract_ingredients_from_text($product_ref); - extract_ingredients_classes_from_text($product_ref); - - compute_languages($product_ref); # need languages for allergens detection - detect_allergens_from_text($product_ref); - - } - - # allergens diffs; - - if ($testing_allergens) { - - my @allergens_import_tags = gen_tags_hierarchy_taxonomy("fr", "allergens", $allergens_import); - - my @allergens_tags = (); - - if (defined $product_ref->{"allergens" . "_hierarchy"}) { - @allergens_tags = @{$product_ref->{"allergens" . "_hierarchy"}}; - - } - else { - print STDERR "allergens_hierarchy field not set\n"; - - } - - my $allergens_import_tags_string = join(", ", @allergens_import_tags); - my $allergens_tags_string = join(", ", @allergens_tags); - - if ($allergens_tags_string ne $allergens_import_tags_string) { - print "ALLERGENS DIFF, code: $code, import: $allergens_import_tags_string\n"; - print "ALLERGENS DIFF, code: $code, detect: $allergens_tags_string\n"; - print "ALLERGENS DIFF 2\t$code\t$allergens_import_tags_string\t$allergens_tags_string\n"; - next if $code eq "3368954600477"; # erreur u - next if $code eq "3256222240480"; # vinegar - next if $code eq "3256226385569"; - next if $code eq "3256220514583"; - next if $code eq "3256222645162"; # weird, to be checked - next if $code eq "3256225051106"; - next if $allergens_import_tags_string =~ /sulphur/; - #exit; - } - - } + $User_id = $editor_user_id; - #"sources": [ - #{ - #"id", "usda-ndb", - #"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available) - #"import_t", "423423" (timestamp of import date) - #"fields" : ["product_name","ingredients","nutrients"] - #"images" : [ "1", "2", "3" ] (images ids) - #}, - #{ - #"id", "usda-ndb", - #"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available) - #"import_t", "523423" (timestamp of import date) - #"fields" : ["ingredients","nutrients"] - #"images" : [ "4", "5", "6" ] (images ids) - #}, + my $response_ref = {}; + analyze_and_enrich_product_data($product_ref, $response_ref); if (not defined $product_ref->{sources}) { $product_ref->{sources} = []; @@ -1674,29 +1569,8 @@ images => \@images_ids, }; - $User_id = $editor_user_id; - if ((not $testing) and (not $testing_allergens)) { - fix_salt_equivalent($product_ref); - - compute_serving_size_data($product_ref); - - compute_nutrition_score($product_ref); - - compute_nova_group($product_ref); - - compute_nutrient_levels($product_ref); - - compute_unknown_nutrients($product_ref); - - ProductOpener::DataQuality::check_quality($product_ref); - - #print STDERR "Storing product code $code\n"; - # use Data::Dumper; - #print STDERR Dumper($product_ref); - #exit; - $product_ref->{owner} = "org-systeme-u"; $product_ref->{owners_tags} = ["org-systeme-u"]; diff --git a/scripts/list_ingredients.pl b/scripts/list_ingredients.pl index 363851b8e3e42..c23887bb980db 100755 --- a/scripts/list_ingredients.pl +++ b/scripts/list_ingredients.pl @@ -53,26 +53,6 @@ use Encode; use JSON::PP; -#use Getopt::Long; -# -#my @fields_to_update = (); -#my $key; -#my $index = ''; -#my $pretend = ''; -#my $process_ingredients = ''; -#my $compute_nutrition_score = ''; -#my $check_quality = ''; -# -#GetOptions ("key=s" => \$key, # string -# "fields=s" => \@fields_to_update, -# "index" => \$index, -# "pretend" => \$pretend, -# "process-ingredients" => \$process_ingredients, -# "compute-nutrition-score" => \$compute_nutrition_score, -# "check-quality" => \$check_quality, -# ) -# or die("Error in command line arguments:\n$\nusage"); - my $query_ref = {}; my $products_collection = get_products_collection(); diff --git a/scripts/revert_changes_from_user.pl b/scripts/revert_changes_from_user.pl index 3053c542e8c38..8a70bdb0b4f61 100755 --- a/scripts/revert_changes_from_user.pl +++ b/scripts/revert_changes_from_user.pl @@ -43,8 +43,6 @@ TXT ; -use CGI::Carp qw(fatalsToBrowser); - use ProductOpener::Config qw/:all/; use ProductOpener::Store qw/:all/; use ProductOpener::Index qw/:all/; @@ -69,13 +67,7 @@ use Getopt::Long; -my @fields_to_update = (); -my $key; -my $index = ''; my $pretend = ''; -my $process_ingredients = ''; -my $compute_nutrition_score = ''; -my $compute_nova = ''; my $reverted_user_id = ''; GetOptions( @@ -91,8 +83,6 @@ $query_ref->{editors_tags} = $reverted_user_id; -print "Update key: $key\n\n"; - my $products_collection = get_products_collection(); my $cursor = $products_collection->query($query_ref)->fields({code => 1}); diff --git a/scripts/update_all_products.pl b/scripts/update_all_products.pl index 1edace0b86a14..27141a741c935 100755 --- a/scripts/update_all_products.pl +++ b/scripts/update_all_products.pl @@ -40,7 +40,7 @@ --query some_field=-some_value match products that don't have some_value for some_field --process-ingredients compute allergens, additives detection --clean-ingredients remove nutrition facts, conservation conditions etc. ---compute-nutrition-score nutriscore +--compute-nutriscore nutriscore --compute-serving-size compute serving size values --compute-history compute history and completeness --check-quality run quality checks @@ -97,7 +97,7 @@ my $process_ingredients = ''; my $process_packagings = ''; my $clean_ingredients = ''; -my $compute_nutrition_score = ''; +my $compute_nutriscore = ''; my $compute_serving_size = ''; my $compute_data_sources = ''; my $compute_nova = ''; @@ -154,7 +154,7 @@ "process-ingredients" => \$process_ingredients, "process-packagings" => \$process_packagings, "assign-categories-properties" => \$assign_categories_properties, - "compute-nutrition-score" => \$compute_nutrition_score, + "compute-nutriscore" => \$compute_nutriscore, "compute-history" => \$compute_history, "compute-serving-size" => \$compute_serving_size, "reassign-energy-kcal" => \$reassign_energy_kcal, @@ -228,7 +228,7 @@ } if ( (not $process_ingredients) - and (not $compute_nutrition_score) + and (not $compute_nutriscore) and (not $compute_nova) and (not $clean_ingredients) and (not $delete_old_fields) @@ -1076,9 +1076,10 @@ compute_nova_group($product_ref); } - if ($compute_nutrition_score) { + if ($compute_nutriscore) { + $product_ref->{misc_tags} = []; fix_salt_equivalent($product_ref); - compute_nutrition_score($product_ref); + compute_nutriscore($product_ref); compute_nutrient_levels($product_ref); } @@ -1197,7 +1198,7 @@ fix_salt_equivalent($product_ref); compute_serving_size_data($product_ref); - compute_nutrition_score($product_ref); + compute_nutriscore($product_ref); compute_nutrient_levels($product_ref); $product_values_changed = 1; } @@ -1220,7 +1221,7 @@ fix_salt_equivalent($product_ref); compute_serving_size_data($product_ref); - compute_nutrition_score($product_ref); + compute_nutriscore($product_ref); compute_nutrient_levels($product_ref); } } diff --git a/stop_words.txt b/stop_words.txt index 76f825a1ef6d1..1e7f99e25737a 100644 --- a/stop_words.txt +++ b/stop_words.txt @@ -125,6 +125,7 @@ LCA Lowercased maltodextrine malus +margarines matche md métal @@ -226,4 +227,4 @@ gzipped webpage webpages bing -txt \ No newline at end of file +txt diff --git a/taxonomies/categories.txt b/taxonomies/categories.txt index 3e82f1f6c5150..8cd575db8a47f 100644 --- a/taxonomies/categories.txt +++ b/taxonomies/categories.txt @@ -67670,10 +67670,6 @@ es:Mostazas violetas fr:Moutardes violettes, Moutardes à la violette nl:Viooltjesmosterds -en:Spices -[0]; my $product_ref = $test_ref->[1]; + # We need salt_value to compute sodium_100g with fix_salt_equivalent + foreach my $prepared ('', '_prepared') { + if (deep_exists($product_ref, "nutriments", "salt${prepared}_100g")) { + $product_ref->{nutriments}{"salt${prepared}_value"} = $product_ref->{nutriments}{"salt${prepared}_100g"}; + } + if (deep_exists($product_ref, "nutriments", "sodium${prepared}_100g")) { + $product_ref->{nutriments}{"sodium${prepared}_value"} + = $product_ref->{nutriments}{"sodium${prepared}_100g"}; + } + } + + fix_salt_equivalent($product_ref); + compute_serving_size_data($product_ref); compute_field_tags($product_ref, $product_ref->{lc}, "categories"); extract_ingredients_from_text($product_ref); special_process_product($product_ref); diag explain compute_estimated_nutrients($product_ref); - compute_nutrition_score($product_ref); + compute_nutriscore($product_ref); compare_to_expected_results($product_ref, "$expected_result_dir/$testid.json", $update_expected_results); } diff --git a/tests/unit/nutriscore_2023.t b/tests/unit/nutriscore_2023.t new file mode 100644 index 0000000000000..96b118c662901 --- /dev/null +++ b/tests/unit/nutriscore_2023.t @@ -0,0 +1,463 @@ +#!/usr/bin/perl -w + +use ProductOpener::PerlStandards; + +use JSON; + +use Test::More; +use Test::Number::Delta relative => 1.001; +use Log::Any::Adapter 'TAP'; + +use ProductOpener::Config qw/:all/; +use ProductOpener::Tags qw/:all/; +use ProductOpener::Food qw/:all/; +use ProductOpener::Ingredients qw/:all/; +use ProductOpener::Nutriscore qw/:all/; +use ProductOpener::NutritionCiqual qw/:all/; +use ProductOpener::NutritionEstimation qw/:all/; +use ProductOpener::Test qw/:all/; + +use Data::DeepAccess qw(deep_exists); + +my ($test_id, $test_dir, $expected_result_dir, $update_expected_results) = (init_expected_results(__FILE__)); + +# Needed to compute estimated nutrients +load_ciqual_data(); + +my @tests = ( + + [ + "cookies", + { + lc => "en", + categories => "cookies", + nutriments => { + energy_100g => 3460, + fat_100g => 90, + "saturated-fat_100g" => 15, + sugars_100g => 0, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + [ + "olive-oil", + { + lc => "en", + categories => "olive oils", + nutriments => { + energy_100g => 3460, + fat_100g => 92, + "saturated-fat_100g" => 14, + sugars_100g => 0, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + [ + "colza-oil", + { + lc => "en", + categories => "colza oils", + nutriments => { + energy_100g => 3760, + fat_100g => 100, + "saturated-fat_100g" => 7, + sugars_100g => 0, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + [ + "walnut-oil", + { + lc => "en", + categories => "walnut oils", + nutriments => { + energy_100g => 3378, + fat_100g => 100, + "saturated-fat_100g" => 10, + sugars_100g => 0, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + [ + "sunflower-oil", + { + lc => "en", + categories => "sunflower oils", + nutriments => { + energy_100g => 3378, + fat_100g => 100, + "saturated-fat_100g" => 10, + sugars_100g => 0, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + + # if no sugar but carbohydrates is 0, consider sugar 0 + [ + "sunflower-oil-no-sugar", + { + lc => "en", + categories => "sunflower oils", + nutriments => { + energy_100g => 3378, + fat_100g => 100, + "saturated-fat_100g" => 10, + carbohydrates_100g => 0, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + + # if no sugar but carbohydrates is 0, consider sugar 0 + # still saturated fat missing will block + [ + "sunflower-oil-no-sugar-no-sat-fat", + { + lc => "en", + categories => "sunflower oils", + nutriments => { + energy_100g => 3378, + fat_100g => 100, + carbohydrates_100g => 0, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + + # saturated fat 1.03 should be rounded to 1.0 which is not strictly greater than 1.0 + [ + "breakfast-cereals", + { + lc => "en", + categories => "breakfast cereals", + nutriments => { + energy_100g => 2450, + fat_100g => 100, + "saturated-fat_100g" => 1.03, + sugars_100g => 31, + salt_100g => 1, + fiber_100g => 6.9, + proteins_100g => 10.3 + } + } + ], + + # dairy drink with milk >= 80% are considered food and not beverages + + [ + "dairy-drinks-without-milk", + { + lc => "en", + categories => "dairy drinks", + nutriments => { + energy_100g => 3378, + fat_100g => 10, + "saturated-fat_100g" => 5, + sugars_100g => 10, + salt_100g => 0, + fiber_100g => 2, + proteins_100g => 5 + }, + ingredients_text => "Water, sugar" + } + ], + [ + "milk", + { + lc => "en", + categories => "milk", + nutriments => { + energy_100g => 3378, + fat_100g => 10, + "saturated-fat_100g" => 5, + sugars_100g => 10, + salt_100g => 0, + fiber_100g => 2, + proteins_100g => 5 + }, + ingredients_text => "Milk" + } + ], + [ + "dairy-drink-with-80-percent-milk", + { + lc => "en", + categories => "dairy drinks", + nutriments => { + energy_100g => 3378, + fat_100g => 10, + "saturated-fat_100g" => 5, + sugars_100g => 10, + salt_100g => 0, + fiber_100g => 2, + proteins_100g => 5 + }, + ingredients_text => "Fresh milk 80%, sugar" + } + ], + [ + "beverage-with-80-percent-milk", + { + lc => "en", + categories => "beverages", + nutriments => { + energy_100g => 3378, + fat_100g => 10, + "saturated-fat_100g" => 5, + sugars_100g => 10, + salt_100g => 0, + fiber_100g => 2, + proteins_100g => 5 + }, + ingredients_text => "Fresh milk 80%, sugar" + } + ], + + [ + "dairy-drink-with-less-than-80-percent-milk", + { + lc => "en", + categories => "dairy drinks", + nutriments => { + energy_100g => 3378, + fat_100g => 10, + "saturated-fat_100g" => 5, + sugars_100g => 10, + salt_100g => 0, + fiber_100g => 2, + proteins_100g => 5 + }, + ingredients_text => "Milk, sugar" + } + ], + + # mushrooms are counted as fruits/vegetables + [ + "mushrooms", + { + lc => "fr", + categories => "meals", + nutriments => { + energy_100g => 667, + fat_100g => 8.4, + "saturated-fat_100g" => 1.2, + sugars_100g => 1.1, + salt_100g => 0.4, + fiber_100g => 10.9, + proteins_100g => 2.4 + }, + ingredients_text => "Pleurotes* 69% (Origine UE), chapelure de mais" + } + ], + + # fruit content indicated at the end of the ingredients list + [ + "fr-gaspacho", + { + lc => "fr", + categories => "gaspachos", + ingredients_text => + "Tomate,concombre,poivron,oignon,eau,huile d'olive vierge extra (1,1%),vinaigre de vin,pain de riz,sel,ail,jus de citron,teneur en légumes: 89%", + nutriments => { + energy_100g => 148, + fat_100g => 10, + "saturated-fat_100g" => 0.2, + sugars_100g => 3, + salt_100g => 0.2, + fiber_100g => 1.1, + proteins_100g => 0.9 + }, + } + + ], + + # if fat is 0 and we have no saturated fat, we consider it 0 + [ + "fr-orange-nectar-0-fat", + { + lc => "en", + categories => "fruit-nectar", + ingredients_text => "Orange 47%, Water, Sugar, Carrots 10%", + nutriments => { + energy_100g => 250, + fat_100g => 0, + sugars_100g => 12, + salt_100g => 0.2, + fiber_100g => 0, + proteins_100g => 0.5 + }, + } + + ], + + # spring waters + ["spring-water-no-nutrition", {lc => "en", categories => "spring water", nutriments => {}}], + ["flavored-spring-water-no-nutrition", {lc => "en", categories => "flavoured spring water", nutriments => {}}], + [ + "flavored-spring-with-nutrition", + { + lc => "en", + categories => "flavoured spring water", + nutriments => { + energy_100g => 378, + fat_100g => 0, + "saturated-fat_100g" => 0, + sugars_100g => 3, + salt_100g => 0, + fiber_100g => 0, + proteins_100g => 0 + } + } + ], + + # Cocoa and chocolate powders + [ + "cocoa-and-chocolate-powders", + { + lc => "en", + "categories" => "cocoa and chocolate powders", + nutriments => { + energy_prepared_100g => 287, + fat_prepared_100g => 0, + "saturated-fat_prepared_100g" => 1.1, + sugars_prepared_100g => 6.3, + salt_prepared_100g => 0.5, + fiber_prepared_100g => 1.9, + proteins_prepared_100g => 3.8 + } + } + ], + + # fruits and vegetables estimates from category or from ingredients + [ + "en-orange-juice-category-and-ingredients", + { + lc => "en", + categories => "orange juices", + ingredients_text => "orange juice 50%, water, sugar", + nutriments => { + energy_100g => 182, + fat_100g => 0, + "saturated-fat_100g" => 0, + sugars_100g => 8.9, + salt_100g => 0.2, + fiber_100g => 0.5, + proteins_100g => 0.2 + }, + } + ], + [ + "en-orange-juice-category", + { + lc => "en", + categories => "orange juices", + nutriments => { + energy_100g => 182, + fat_100g => 0, + "saturated-fat_100g" => 0, + sugars_100g => 8.9, + salt_100g => 0.2, + fiber_100g => 0.5, + proteins_100g => 0.2 + }, + } + ], + + # categories without Nutri-Score + + [ + "en-beers-category", + { + lc => "en", + categories => "beers", + nutriments => { + energy_100g => 182, + fat_100g => 0, + "saturated-fat_100g" => 0, + sugars_100g => 8.9, + salt_100g => 0.2, + fiber_100g => 0.5, + proteins_100g => 0.2 + }, + } + ], + + # Nutri-Score from estimated nutrients + [ + "en-sugar-estimated-nutrients", + { + lc => "en", + categories => "sugars", + ingredients_text => "sugar", + } + ], + [ + "en-apple-estimated-nutrients", + { + lc => "en", + categories => "apples", + ingredients_text => "apples", + } + ], + [ + "94-percent-sugar-and-unknown-ingredient", + { + lc => "en", + categories => "sugars", + ingredients_text => "sugar 94%, strange ingredient", + } + ], + +); + +my $json = JSON->new->allow_nonref->canonical; + +foreach my $test_ref (@tests) { + + my $testid = $test_ref->[0]; + my $product_ref = $test_ref->[1]; + + # We need salt_value to compute sodium_100g with fix_salt_equivalent + foreach my $prepared ('', '_prepared') { + if (deep_exists($product_ref, "nutriments", "salt${prepared}_100g")) { + $product_ref->{nutriments}{"salt${prepared}_value"} = $product_ref->{nutriments}{"salt${prepared}_100g"}; + } + if (deep_exists($product_ref, "nutriments", "sodium${prepared}_100g")) { + $product_ref->{nutriments}{"sodium${prepared}_value"} + = $product_ref->{nutriments}{"sodium${prepared}_100g"}; + } + } + + fix_salt_equivalent($product_ref); + compute_serving_size_data($product_ref); + compute_field_tags($product_ref, $product_ref->{lc}, "categories"); + extract_ingredients_from_text($product_ref); + special_process_product($product_ref); + compute_estimated_nutrients($product_ref); + compute_nutriscore($product_ref, "2023"); + + compare_to_expected_results($product_ref, "$expected_result_dir/$testid.json", $update_expected_results); +} + +is(compute_nutriscore_grade(1.56, 1, 0), "c"); + +done_testing();