refactor: specific processes depending on product type (#10171)

cgi/product_multilingual.pl

@@ -37,7 +37,7 @@
 use ProductOpener::Mail qw/send_email_to_admin/;
 use ProductOpener::Products qw/:all/;
 use ProductOpener::Food
-	qw/%nutriments_tables %other_nutriments_lists assign_nutriments_values_from_request_parameters compute_serving_size_data get_nutrient_unit/;
+	qw/%nutriments_tables %other_nutriments_lists assign_nutriments_values_from_request_parameters compute_nutrition_data_per_100g_and_per_serving get_nutrient_unit/;
 use ProductOpener::Units qw/g_to_unit mmoll_to_unit/;
 use ProductOpener::Ingredients qw/:all/;
 use ProductOpener::Images qw/:all/;
@@ -763,7 +763,7 @@ ($product_ref, $field, $language)
 if (($action eq 'display') and (($type eq 'add') or ($type eq 'edit'))) {
 
 	# Populate the energy-kcal or energy-kj field from the energy field if it exists
-	compute_serving_size_data($product_ref);
+	compute_nutrition_data_per_100g_and_per_serving($product_ref);
 
 	my $template_data_ref_display = {};
 

cgi/test_ingredients_analysis.pl

@@ -32,7 +32,7 @@
 use ProductOpener::Lang qw/$lc/;
 use ProductOpener::Tags qw/:all/;
 use ProductOpener::Ingredients
-	qw/clean_ingredients_text extract_ingredients_classes_from_text extract_ingredients_from_text preparse_ingredients_text/;
+	qw/clean_ingredients_text extract_additives_from_text extract_ingredients_from_text preparse_ingredients_text/;
 use ProductOpener::Text qw/remove_tags_and_quote/;
 
 use CGI qw/:cgi :form escapeHTML charset/;
@@ -71,8 +71,8 @@
 	clean_ingredients_text($product_ref);
 	$log->debug("extract_ingredients_from_text") if $log->is_debug();
 	extract_ingredients_from_text($product_ref);
-	$log->debug("extract_ingredients_classes_from_text") if $log->is_debug();
-	extract_ingredients_classes_from_text($product_ref);
+	$log->debug("extract_additives_from_text") if $log->is_debug();
+	extract_additives_from_text($product_ref);
 
 	my $html_details = display_ingredients_analysis_details($product_ref);
 	$html_details =~ s/.*tabindex="-1">/<div>/;

lib/ProductOpener/BeautyProducts.pm

@@ -0,0 +1,84 @@
+# This file is part of Product Opener.
+#
+# Product Opener
+# Copyright (C) 2011-2023 Association Open Food Facts
+# Contact: [email protected]
+# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France
+#
+# Product Opener is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+=encoding UTF-8
+
+=head1 NAME
+
+ProductOpener::BeautyProducts - functions related to beauty products and nutrition
+
+=head1 DESCRIPTION
+
+C<ProductOpener::BeautyProducts> contains functions specific to beauty products.
+
+..
+
+=cut
+
+package ProductOpener::BeautyProducts;
+
+use ProductOpener::PerlStandards;
+use Exporter qw< import >;
+
+BEGIN {
+	use vars qw(@ISA @EXPORT_OK %EXPORT_TAGS);
+	@EXPORT_OK = qw(
+
+		&specific_processes_for_beauty_product
+
+	);    # symbols to export on request
+	%EXPORT_TAGS = (all => [@EXPORT_OK]);
+}
+
+use vars @EXPORT_OK;
+
+use ProductOpener::Ingredients
+	qw/select_ingredients_lc clean_ingredients_text extract_ingredients_from_text extract_additives_from_text detect_allergens_from_text/;
+use ProductOpener::Food qw/compute_nutrition_data_per_100g_and_per_serving assign_categories_properties_to_product/;
+
+use Log::Any qw($log);
+
+=head2 specific_processes_for_beauty_product ( $ingredients_ref )
+
+Runs specific processes for beauty products:
+
+- Ingredients analysis
+- Additives detection
+- Allergens detection
+- Computation of scores
+
+=cut
+
+sub specific_processes_for_beauty_product ($product_ref) {
+
+	# Ingredients analysis
+
+	# Select best language to parse ingredients
+	$product_ref->{ingredients_lc} = select_ingredients_lc($product_ref);
+	clean_ingredients_text($product_ref);
+	extract_ingredients_from_text($product_ref);
+
+	# Serving size
+	compute_nutrition_data_per_100g_and_per_serving($product_ref);
+
+	return;
+}
+
+1;

lib/ProductOpener/Config_obf.pm

@@ -248,7 +248,7 @@ HTML
 # fields for which we will load taxonomies
 
 @taxonomy_fields
-	= qw(units states countries languages labels categories additives allergens traces nutrient_levels ingredients periods_after_opening inci_functions);
+	= qw(units states countries languages labels categories additives additives_classes allergens traces nutrient_levels ingredients periods_after_opening inci_functions);
 
 # tag types (=facets) that should be indexed by web crawlers, all other tag types are not indexable
 @index_tag_types = qw(brands categories labels additives products);

lib/ProductOpener/Display.pm

@@ -4376,8 +4376,7 @@ sub display_search_results ($request_ref) {
 	$current_link =~ s/^\&/\?/;
 	$current_link = "/search" . $current_link;
 
-	if ((defined single_param("user_preferences")) and (single_param("user_preferences")) and not($request_ref->{api}))
-	{
+	if (not($request_ref->{api})) {
 
 		# The results will be filtered and ranked on the client side
 
@@ -4919,7 +4918,7 @@ sub search_and_display_products ($request_ref, $query_ref, $sort_by, $limit, $pa
 		$limit = $request_ref->{page_size};
 	}
 	# If user preferences are turned on, return 100 products per page
-	elsif ((not defined $request_ref->{api}) and ($request_ref->{user_preferences})) {
+	elsif (not defined $request_ref->{api}) {
 		$limit = 100;
 	}
 	else {
@@ -5083,7 +5082,6 @@ sub search_and_display_products ($request_ref, $query_ref, $sort_by, $limit, $pa
 	{
 		$fields_ref = {};
 	}
-	# - if we use user preferences, we need a lot of fields to compute product attributes: load them all
 	elsif ($request_ref->{user_preferences}) {
 		# we restrict the fields that are queried to MongoDB, and use the basic ones and those necessary
 		# by Attributes.pm to compute attributes.
@@ -5358,7 +5356,7 @@ sub search_and_display_products ($request_ref, $query_ref, $sort_by, $limit, $pa
 		# For non API queries with user preferences, we need to add attributes
 		# For non API queries, we need to compute attributes for personal search
 		my $fields;
-		if ((not defined $request_ref->{api}) and ($request_ref->{user_preferences})) {
+		if (not defined $request_ref->{api}) {
 			$fields = "code,product_display_name,url,image_front_small_url,attribute_groups";
 		}
 		else {
@@ -5412,49 +5410,44 @@ sub search_and_display_products ($request_ref, $query_ref, $sort_by, $limit, $pa
 		$html =~ s/(href|src)=("\/)/$1="$formatted_subdomain\//g;
 	}
 
-	if ($request_ref->{user_preferences}) {
+	my $preferences_text = sprintf(lang("classify_the_d_products_below_according_to_your_preferences"), $page_count);
 
-		my $preferences_text
-			= sprintf(lang("classify_the_d_products_below_according_to_your_preferences"), $page_count);
+	my $products_json = '[]';
 
-		my $products_json = '[]';
+	if (defined $request_ref->{structured_response}{products}) {
+		$products_json = $json->encode($request_ref->{structured_response}{products});
+	}
 
-		if (defined $request_ref->{structured_response}{products}) {
-			$products_json = $json->encode($request_ref->{structured_response}{products});
+	my $contributor_prefs_json = $json->encode(
+		{
+			display_barcode => $User{display_barcode},
+			edit_link => $User{edit_link},
 		}
+	);
 
-		my $contributor_prefs_json = $json->encode(
-			{
-				display_barcode => $User{display_barcode},
-				edit_link => $User{edit_link},
-			}
-		);
-
-		$scripts .= <<JS
+	$scripts .= <<JS
 <script type="text/javascript">
 var page_type = "products";
 var preferences_text = "$preferences_text";
 var contributor_prefs = $contributor_prefs_json;
 var products = $products_json;
 </script>
 JS
-			;
+		;
 
-		$scripts .= <<JS
+	$scripts .= <<JS
 <script src="$static_subdomain/js/product-preferences.js"></script>
 <script src="$static_subdomain/js/product-search.js"></script>
 JS
-			;
+		;
 
-		$initjs .= <<JS
+	$initjs .= <<JS
 display_user_product_preferences("#preferences_selected", "#preferences_selection_form", function () {
 	rank_and_display_products("#search_results", products, contributor_prefs);
 });
 rank_and_display_products("#search_results", products, contributor_prefs);
 JS
-			;
-
-	}
+		;
 
 	process_template('web/common/includes/list_of_products.tt.html', $template_data_ref, \$html)
 		|| return "template error: " . $tt->error();
@@ -7690,6 +7683,9 @@ JS
 			= display_knowledge_panel($product_ref, $product_ref->{"knowledge_panels_" . $lc}, "contribution_card");
 	}
 
+	# User preferences
+	$template_data_ref->{user_preferences} = $request_ref->{user_preferences};
+
 	# The front product image is rendered with the same template as the ingredients, nutrition and packaging images
 	# that are displayed directly through the knowledge panels
 	$template_data_ref->{front_image} = data_to_display_image($product_ref, "front", $lc);

lib/ProductOpener/Ecoscore.pm

@@ -75,6 +75,9 @@ use Text::CSV();
 use Math::Round;
 use Data::DeepAccess qw(deep_get deep_exists);
 
+my $agribalyse_data_loaded = 0;
+my $ecoscore_data_loaded = 0;
+
 %agribalyse = ();
 
 =head1 VARIABLES
@@ -107,6 +110,7 @@ Loads the AgriBalyse database.
 =cut
 
 sub load_agribalyse_data() {
+
 	my $agribalyse_details_by_step_csv_file = $data_root . "/external-data/ecoscore/agribalyse/AGRIBALYSE_vf.csv.2";
 
 	my $rows_ref = [];
@@ -165,6 +169,9 @@ sub load_agribalyse_data() {
 	else {
 		die("Could not open agribalyse CSV $agribalyse_details_by_step_csv_file: $!");
 	}
+
+	$agribalyse_data_loaded = 1;
+
 	return;
 }
 
@@ -665,6 +672,8 @@ sub load_ecoscore_data() {
 
 	load_ecoscore_data_origins_of_ingredients();
 	load_ecoscore_data_packaging();
+
+	$ecoscore_data_loaded = 1;
 	return;
 }
 
@@ -689,6 +698,15 @@ Returned values:
 =cut
 
 sub compute_ecoscore ($product_ref) {
+
+	# Some test cases do not load the Eco-Score data (e.g. food.t) as they don't test the Eco-Score
+	# but compute_ecoscore() is still called by specific_processes_for_food_product($product_ref);
+	# So we check if the data is loaded, and do not compute the Eco-Score if not loaded
+	if (not($ecoscore_data_loaded and $agribalyse_data_loaded)) {
+		$log->warn("Eco-Score data not loaded, cannot compute Eco-Score") if $log->is_warn();
+		return;
+	}
+
 	my $old_ecoscore_data = $product_ref->{ecoscore_data};
 	my $old_agribalyse = $old_ecoscore_data->{agribalyse};
 	my $old_ecoscore_grade = $old_ecoscore_data->{grade};

lib/ProductOpener/Food.pm

@@ -69,15 +69,13 @@ BEGIN {
 		&compute_nutriscore_data
 		&compute_nutriscore
 		&compute_nova_group
-		&compute_serving_size_data
+		&compute_nutrition_data_per_100g_and_per_serving
 		&compute_unknown_nutrients
 		&compute_nutrient_levels
 		&compute_estimated_nutrients
 
 		&compare_nutriments
 
-		&special_process_product
-
 		&extract_nutrition_from_image
 
 		&default_unit_for_nid
@@ -1212,23 +1210,6 @@ sub is_red_meat_product_for_nutrition_score ($product_ref) {
 	return 0;
 }
 
-=head2 special_process_product ( $ingredients_ref )
-
-Computes food groups, and whether a product is to be considered a beverage for the Nutri-Score.
-
-Ingredients analysis (extract_ingredients_from_text) needs to be done before calling this function?
-
-=cut
-
-sub special_process_product ($product_ref) {
-
-	assign_categories_properties_to_product($product_ref);
-
-	compute_food_groups($product_ref);
-
-	return;
-}
-
 sub fix_salt_equivalent ($product_ref) {
 
 	# EU fixes the conversion: sodium = salt / 2.5 (instead of 2.54 previously)
@@ -2186,45 +2167,19 @@ sub compute_nutriscore ($product_ref, $current_version = "2021") {
 	return;
 }
 
-sub compute_serving_size_data ($product_ref) {
-
-	# identify products that do not have comparable nutrition data
-	# e.g. products with multiple nutrition facts tables
-	# except in some cases like breakfast cereals
-	# bug #1145
-	# old
+=head2 compute_nutrition_data_per_100g_and_per_serving ($product_ref)
 
-	# Delete old fields
-	(defined $product_ref->{not_comparable_nutrition_data}) and delete $product_ref->{not_comparable_nutrition_data};
-	(defined $product_ref->{multiple_nutrition_data}) and delete $product_ref->{multiple_nutrition_data};
+Input nutrition data is indicated per 100g or per serving.
+This function computes the nutrition data for the other quantity (per serving or per 100g) if we know the serving quantity.
 
-	(defined $product_ref->{product_quantity}) and delete $product_ref->{product_quantity};
-	(defined $product_ref->{product_quantity_unit}) and delete $product_ref->{product_quantity_unit};
-	if ((defined $product_ref->{quantity}) and ($product_ref->{quantity} ne "")) {
-		my $product_quantity = normalize_quantity($product_ref->{quantity});
-		if (defined $product_quantity) {
-			$product_ref->{product_quantity} = $product_quantity;
-		}
-		my $product_quantity_unit = extract_standard_unit($product_ref->{quantity});
-		if (defined $product_quantity_unit) {
-			$product_ref->{product_quantity_unit} = $product_quantity_unit;
-		}
-	}
+=cut
 
-	if ((defined $product_ref->{serving_size}) and ($product_ref->{serving_size} ne "")) {
-		$product_ref->{serving_quantity} = normalize_serving_size($product_ref->{serving_size});
+sub compute_nutrition_data_per_100g_and_per_serving ($product_ref) {
 
-		my $serving_quantity_unit = extract_standard_unit($product_ref->{serving_size});
-		if (defined $serving_quantity_unit) {
-			$product_ref->{serving_quantity_unit} = $serving_quantity_unit;
-		}
-	}
-	else {
-		(defined $product_ref->{serving_quantity}) and delete $product_ref->{serving_quantity};
-		(defined $product_ref->{serving_size})
-			and ($product_ref->{serving_size} eq "")
-			and delete $product_ref->{serving_size};
-	}
+	# Make sure we have normalized the product quantity and the serving size
+	# in a normal setting, this function has already been called by analyze_and_enrich_product_data()
+	# but some test functions (e.g. in food.t) may call this function directly
+	normalize_product_quantity_and_serving_size($product_ref);
 
 	# Record if we have nutrient values for as sold or prepared types,
 	# so that we can check the nutrition_data and nutrition_data_prepared boxes if we have data