feat: New Nutri-Score formula (v2) for food (2022) and beverages (202…

…3) - ready for review (#8832)
openfoodfacts · Aug 29, 2023 · 8ce0abc · 8ce0abc
1 parent 2c126ca
commit 8ce0abc
Show file tree

Hide file tree

Showing 162 changed files with 15,462 additions and 790 deletions.
diff --git a/lib/ProductOpener/Food.pm b/lib/ProductOpener/Food.pm
diff --git a/lib/ProductOpener/Import.pm b/lib/ProductOpener/Import.pm
@@ -3027,7 +3027,7 @@ sub import_products_categories_from_public_database ($args_ref) {
 						$log->debug("Food::special_process_product") if $log->is_debug();
 						ProductOpener::Food::special_process_product($product_ref);
 					}
-					compute_nutrition_score($product_ref);
+					compute_nutriscore($product_ref);
 					compute_nova_group($product_ref);
 					compute_nutrient_levels($product_ref);
 					compute_unknown_nutrients($product_ref);

diff --git a/lib/ProductOpener/Nutriscore.pm b/lib/ProductOpener/Nutriscore.pm
diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm
@@ -3529,6 +3529,9 @@ sub analyze_and_enrich_product_data ($product_ref, $response_ref) {
 
 	$log->debug("analyze_and_enrich_product_data - start") if $log->is_debug();
 
+	# Initialiaze the misc_tags, they will be populated by functions called by this function
+	$product_ref->{misc_tags} = [];
+
 	if (    (defined $product_ref->{nutriments}{"carbon-footprint"})
 		and ($product_ref->{nutriments}{"carbon-footprint"} ne ''))
 	{
@@ -3577,7 +3580,7 @@ sub analyze_and_enrich_product_data ($product_ref, $response_ref) {
 
 	compute_estimated_nutrients($product_ref);
 
-	compute_nutrition_score($product_ref);
+	compute_nutriscore($product_ref);
 
 	compute_nova_group($product_ref);
 

diff --git a/po/tags/en.po b/po/tags/en.po
@@ -590,3 +590,32 @@ msgctxt "packaging_recycling:singular"
 msgid "packaging-recycling"
 msgstr "packaging-recycling"
 
+# Please do not translate nutri-score
+msgctxt "nutriscore:plural"
+msgid "nutri-score"
+msgstr "nutri-score"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore:singular"
+msgid "nutri-score"
+msgstr "nutri-score"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2021:plural"
+msgid "nutri-score-2021"
+msgstr "nutri-score-2021"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2021:singular"
+msgid "nutri-score-2021"
+msgstr "nutri-score-2021"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2023:plural"
+msgid "nutri-score-2023"
+msgstr "nutri-score-2023"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2023:singular"
+msgid "nutri-score-2023"
+msgstr "nutri-score-2023"
diff --git a/po/tags/tags.pot b/po/tags/tags.pot
@@ -694,3 +694,32 @@ msgctxt "packaging_recycling:singular"
 msgid "packaging-recycling"
 msgstr "packaging-recycling"
 
+# Please do not translate nutri-score
+msgctxt "nutriscore:plural"
+msgid "nutri-score"
+msgstr "nutri-score"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore:singular"
+msgid "nutri-score"
+msgstr "nutri-score"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2021:plural"
+msgid "nutri-score-2021"
+msgstr "nutri-score-2021"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2021:singular"
+msgid "nutri-score-2021"
+msgstr "nutri-score-2021"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2023:plural"
+msgid "nutri-score-2023"
+msgstr "nutri-score-2023"
+
+# Please do not translate nutri-score
+msgctxt "nutriscore_2023:singular"
+msgid "nutri-score-2023"
+msgstr "nutri-score-2023"
diff --git a/scripts/aggregate_ingredients.pl b/scripts/aggregate_ingredients.pl
@@ -54,26 +54,6 @@
 use Encode;
 use JSON::PP;
 
-#use Getopt::Long;
-#
-#my @fields_to_update = ();
-#my $key;
-#my $index = '';
-#my $pretend = '';
-#my $process_ingredients = '';
-#my $compute_nutrition_score = '';
-#my $check_quality = '';
-#
-#GetOptions ("key=s"   => \$key,      # string
-#			"fields=s" => \@fields_to_update,
-#			"index" => \$index,
-#			"pretend" => \$pretend,
-#			"process-ingredients" => \$process_ingredients,
-#			"compute-nutrition-score" => \$compute_nutrition_score,
-#			"check-quality" => \$check_quality,
-#			)
-# or die("Error in command line arguments:\n$\nusage");
-
 my $query_ref = {};
 
 my $cursor = get_products_collection()->query($query_ref)->fields({code => 1});

diff --git a/scripts/extract_individual_ingredients.pl b/scripts/extract_individual_ingredients.pl
@@ -54,26 +54,6 @@
 use Encode;
 use JSON::PP;
 
-#use Getopt::Long;
-#
-#my @fields_to_update = ();
-#my $key;
-#my $index = '';
-#my $pretend = '';
-#my $process_ingredients = '';
-#my $compute_nutrition_score = '';
-#my $check_quality = '';
-#
-#GetOptions ("key=s"   => \$key,      # string
-#			"fields=s" => \@fields_to_update,
-#			"index" => \$index,
-#			"pretend" => \$pretend,
-#			"process-ingredients" => \$process_ingredients,
-#			"compute-nutrition-score" => \$compute_nutrition_score,
-#			"check-quality" => \$check_quality,
-#			)
-# or die("Error in command line arguments:\n$\nusage");
-
 my $query_ref = {};
 
 my $products_collection = get_products_collection();

diff --git a/scripts/import_fleurymichon.pl b/scripts/import_fleurymichon.pl
@@ -877,70 +877,10 @@
 	}
 	#exit;
 
-	# Process the fields
-
-	# Food category rules for sweeetened/sugared beverages
-	# French PNNS groups from categories
-
-	if ($server_domain =~ /openfoodfacts/) {
-		ProductOpener::Food::special_process_product($product_ref);
-	}
-
-	if (    (defined $product_ref->{nutriments}{"carbon-footprint"})
-		and ($product_ref->{nutriments}{"carbon-footprint"} ne ''))
-	{
-		push @{$product_ref->{"labels_hierarchy"}}, "en:carbon-footprint";
-		push @{$product_ref->{"labels_tags"}}, "en:carbon-footprint";
-	}
-
-	if ((defined $product_ref->{nutriments}{"glycemic-index"}) and ($product_ref->{nutriments}{"glycemic-index"} ne ''))
-	{
-		push @{$product_ref->{"labels_hierarchy"}}, "en:glycemic-index";
-		push @{$product_ref->{"labels_tags"}}, "en:glycemic-index";
-	}
-
-	# Language and language code / subsite
-
-	if (defined $product_ref->{lang}) {
-		$product_ref->{lc} = $product_ref->{lang};
-	}
-
-	if (not defined $lang_lc{$product_ref->{lc}}) {
-		$product_ref->{lc} = 'xx';
-	}
-
-	# For fields that can have different values in different languages, copy the main language value to the non suffixed field
-
-	foreach my $field (keys %language_fields) {
-		if ($field !~ /_image/) {
-			if (defined $product_ref->{$field . "_$product_ref->{lc}"}) {
-				$product_ref->{$field} = $product_ref->{$field . "_$product_ref->{lc}"};
-			}
-		}
-	}
-
-	# Ingredients classes
-	extract_ingredients_from_text($product_ref);
-	extract_ingredients_classes_from_text($product_ref);
-
-	compute_languages($product_ref);    # need languages for allergens detection
-	detect_allergens_from_text($product_ref);
+	$User_id = $editor_user_id;
 
-	#"sources": [
-	#{
-	#"id", "usda-ndb",
-	#"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available)
-	#"import_t", "423423" (timestamp of import date)
-	#"fields" : ["product_name","ingredients","nutrients"]
-	#"images" : [ "1", "2", "3" ] (images ids)
-	#},
-	#{
-	#"id", "usda-ndb",
-	#"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available)
-	#"import_t", "523423" (timestamp of import date)
-	#"fields" : ["ingredients","nutrients"]
-	#"images" : [ "4", "5", "6" ] (images ids)
-	#},
+	my $response_ref = {};
+	analyze_and_enrich_product_data($product_ref, $response_ref);
 
 	if (not defined $product_ref->{sources}) {
 		$product_ref->{sources} = [];
@@ -957,25 +897,8 @@
 		images => \@images_ids,
 		};
 
-	$User_id = $editor_user_id;
-
 	if (not $testing) {
 
-		fix_salt_equivalent($product_ref);
-
-		compute_serving_size_data($product_ref);
-
-		compute_nutrition_score($product_ref);
-
-		compute_nutrient_levels($product_ref);
-
-		compute_unknown_nutrients($product_ref);
-
-		#print STDERR "Storing product code $code\n";
-		#				use Data::Dumper;
-		#print STDERR Dumper($product_ref);
-		#exit;
-
 		store_product($User_id, $product_ref, "Editing product (import_fleurymichon_ch.pl bulk import) - " . $comment);
 
 		push @edited, $code;

diff --git a/scripts/import_systemeu.pl b/scripts/import_systemeu.pl
@@ -1549,115 +1549,10 @@
 
 	# Process the fields
 
-	# Food category rules for sweeetened/sugared beverages
-	# French PNNS groups from categories
-
-	if ($server_domain =~ /openfoodfacts/) {
-		ProductOpener::Food::special_process_product($product_ref);
-	}
-
-	if (    (defined $product_ref->{nutriments}{"carbon-footprint"})
-		and ($product_ref->{nutriments}{"carbon-footprint"} ne ''))
-	{
-		push @{$product_ref->{"labels_hierarchy"}}, "en:carbon-footprint";
-		push @{$product_ref->{"labels_tags"}}, "en:carbon-footprint";
-	}
-
-	if ((defined $product_ref->{nutriments}{"glycemic-index"}) and ($product_ref->{nutriments}{"glycemic-index"} ne ''))
-	{
-		push @{$product_ref->{"labels_hierarchy"}}, "en:glycemic-index";
-		push @{$product_ref->{"labels_tags"}}, "en:glycemic-index";
-	}
-
-	# Language and language code / subsite
-
-	if (defined $product_ref->{lang}) {
-		$product_ref->{lc} = $product_ref->{lang};
-	}
-
-	# For fields that can have different values in different languages, copy the main language value to the non suffixed field
-
-	foreach my $field (keys %language_fields) {
-		if ($field !~ /_image/) {
-			if (defined $product_ref->{$field . "_$product_ref->{lc}"}) {
-				$product_ref->{$field} = $product_ref->{$field . "_$product_ref->{lc}"};
-			}
-		}
-	}
-
-	if ($testing_allergens) {
-
-		$product_ref->{allergens} = "";
-		$product_ref->{traces} = "";
-	}
-
-	if ($server_domain =~ /openfoodfacts/) {
-		ProductOpener::Food::special_process_product($product_ref);
-	}
-
-	if (($testing_allergens) or (not $testing)) {
-		# Ingredients classes
-		print STDERR "computing allergens etc.\n";
-
-		extract_ingredients_from_text($product_ref);
-		extract_ingredients_classes_from_text($product_ref);
-
-		compute_languages($product_ref);    # need languages for allergens detection
-		detect_allergens_from_text($product_ref);
-
-	}
-
-	# allergens diffs;
-
-	if ($testing_allergens) {
-
-		my @allergens_import_tags = gen_tags_hierarchy_taxonomy("fr", "allergens", $allergens_import);
-
-		my @allergens_tags = ();
-
-		if (defined $product_ref->{"allergens" . "_hierarchy"}) {
-			@allergens_tags = @{$product_ref->{"allergens" . "_hierarchy"}};
-
-		}
-		else {
-			print STDERR "allergens_hierarchy field not set\n";
-
-		}
-
-		my $allergens_import_tags_string = join(", ", @allergens_import_tags);
-		my $allergens_tags_string = join(", ", @allergens_tags);
-
-		if ($allergens_tags_string ne $allergens_import_tags_string) {
-			print "ALLERGENS DIFF, code: $code, import: $allergens_import_tags_string\n";
-			print "ALLERGENS DIFF, code: $code, detect: $allergens_tags_string\n";
-			print "ALLERGENS DIFF 2\t$code\t$allergens_import_tags_string\t$allergens_tags_string\n";
-			next if $code eq "3368954600477";    # erreur u
-			next if $code eq "3256222240480";    # vinegar
-			next if $code eq "3256226385569";
-			next if $code eq "3256220514583";
-			next if $code eq "3256222645162";    # weird, to be checked
-			next if $code eq "3256225051106";
-			next if $allergens_import_tags_string =~ /sulphur/;
-			#exit;
-		}
-
-	}
+	$User_id = $editor_user_id;
 
-	#"sources": [
-	#{
-	#"id", "usda-ndb",
-	#"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available)
-	#"import_t", "423423" (timestamp of import date)
-	#"fields" : ["product_name","ingredients","nutrients"]
-	#"images" : [ "1", "2", "3" ] (images ids)
-	#},
-	#{
-	#"id", "usda-ndb",
-	#"url", "https://ndb.nal.usda.gov/ndb/foods/show/58513?format=Abridged&reportfmt=csv&Qv=1" (direct product url if available)
-	#"import_t", "523423" (timestamp of import date)
-	#"fields" : ["ingredients","nutrients"]
-	#"images" : [ "4", "5", "6" ] (images ids)
-	#},
+	my $response_ref = {};
+	analyze_and_enrich_product_data($product_ref, $response_ref);
 
 	if (not defined $product_ref->{sources}) {
 		$product_ref->{sources} = [];
@@ -1674,29 +1569,8 @@
 		images => \@images_ids,
 		};
 
-	$User_id = $editor_user_id;
-
 	if ((not $testing) and (not $testing_allergens)) {
 
-		fix_salt_equivalent($product_ref);
-
-		compute_serving_size_data($product_ref);
-
-		compute_nutrition_score($product_ref);
-
-		compute_nova_group($product_ref);
-
-		compute_nutrient_levels($product_ref);
-
-		compute_unknown_nutrients($product_ref);
-
-		ProductOpener::DataQuality::check_quality($product_ref);
-
-		#print STDERR "Storing product code $code\n";
-		#				use Data::Dumper;
-		#print STDERR Dumper($product_ref);
-		#exit;
-
 		$product_ref->{owner} = "org-systeme-u";
 		$product_ref->{owners_tags} = ["org-systeme-u"];