feat: Statistics for packagings materials by categories (#8613)

Makefile

@@ -286,6 +286,8 @@ clean_tests:
 update_tests_results:
 	@echo "🥫 Updated expected test results with actuals for easy Git diff"
 	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront incron
+	${DOCKER_COMPOSE_TEST} run --no-deps --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} backend /opt/product-opener/scripts/build_tags_taxonomy.pl ${name}
+	${DOCKER_COMPOSE_TEST} run --rm backend perl -I/opt/product-opener/lib -I/opt/perl/local/lib/perl5 /opt/product-opener/scripts/build_lang.pl
 	${DOCKER_COMPOSE_TEST} exec -T -w /opt/product-opener/tests backend bash update_tests_results.sh
 	${DOCKER_COMPOSE_TEST} stop
 

lib/ProductOpener/Display.pm

@@ -162,6 +162,7 @@ use ProductOpener::Nutriscore qw(:all);
 use ProductOpener::Ecoscore qw(:all);
 use ProductOpener::Attributes qw(:all);
 use ProductOpener::KnowledgePanels qw(:all);
+use ProductOpener::KnowledgePanelsTags qw(:all);
 use ProductOpener::Orgs qw(:all);
 use ProductOpener::Web qw(:all);
 use ProductOpener::Recipes qw(:all);
@@ -4057,6 +4058,18 @@ HTML
 				if ($description ne "") {
 					$tag_template_data_ref->{description} = $description;
 				}
+
+				# Display knowledge panels for the tag, if any
+
+				initialize_knowledge_panels_options($knowledge_panels_options_ref, $request_ref);
+				my $tag_ref = {};    # Object to store the knowledge panels
+				my $panels_created
+					= create_tag_knowledge_panels($tag_ref, $lc, $cc, $knowledge_panels_options_ref, $tagtype,
+					$canon_tagid);
+				if ($panels_created) {
+					$tag_template_data_ref->{tag_panels}
+						= display_knowledge_panel($tag_ref, $tag_ref->{"knowledge_panels_" . $lc}, "root");
+				}
 			}
 
 			$tag_template_data_ref->{products_title} = $products_title;

lib/ProductOpener/KnowledgePanelsTags.pm

@@ -0,0 +1,175 @@
+# This file is part of Product Opener.
+#
+# Product Opener
+# Copyright (C) 2011-2023 Association Open Food Facts
+# Contact: [email protected]
+# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France
+#
+# Product Opener is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+=head1 NAME
+
+ProductOpener::KnowledgePanelsContribution - Generate knowledge panels around contribution
+
+=head1 SYNOPSIS
+
+This is a subpart of Knowledge Panels where we concentrate around contribution information
+
+=cut
+
+package ProductOpener::KnowledgePanelsTags;
+
+use ProductOpener::PerlStandards;
+use Exporter qw< import >;
+
+use Log::Any qw($log);
+
+BEGIN {
+	use vars qw(@ISA @EXPORT_OK %EXPORT_TAGS);
+	@EXPORT_OK = qw(
+		&create_tag_knowledge_panels
+		&create_category_packagings_materials_panel
+	);    # symbols to export on request
+	%EXPORT_TAGS = (all => [@EXPORT_OK]);
+}
+
+use vars @EXPORT_OK;
+
+use ProductOpener::KnowledgePanels qw(create_panel_from_json_template);
+use ProductOpener::Tags qw(:all);
+use ProductOpener::Packaging qw(:all);
+
+use Encode;
+use Data::DeepAccess qw(deep_get);
+
+=head2 create_tag_knowledge_panels ($tag_ref, $target_lc, $target_cc, $options_ref, $tagtype, $canon_tagid)
+
+Create all knowledge panels for a tag, with strings (descriptions, recommendations etc.)
+in a specific language, and return them in an array of panels.
+
+=head3 Arguments
+
+=head4 tag reference $tag_ref
+
+Hash structure for the tag. Can be empty. Will be used to return the panels.
+
+=head4 language code $target_lc (or "data")
+
+Returned panels contain both data and strings intended to be displayed to users.
+This parameter sets the desired language for the user facing strings.
+
+If $target_lc is equal to "data", no strings are returned.
+
+=head4 country code $target_cc
+
+Needed for some country specific panels like the Eco-Score.
+
+=head4 options $options_ref
+
+Defines how some panels should be created (or not created)
+
+- deactivate_[panel_id] : do not create a default panel -- currently unimplemented
+- activate_[panel_id] : create an on demand panel -- currently only for physical_activities panel
+
+=head4 tag type $tagtype
+
+=head4 canonical tagid $canon_tagid
+
+=head3 Return values
+
+Panels are returned in the "knowledge_panels_[$target_lc]" hash of the tag reference
+passed as input.
+
+=cut
+
+sub create_tag_knowledge_panels ($tag_ref, $target_lc, $target_cc, $options_ref, $tagtype, $canon_tagid) {
+
+	$log->debug("create knowledge panels for tag",
+		{tagtype => $tagtype, tagid => $canon_tagid, target_lc => $target_lc})
+		if $log->is_debug();
+
+	# Initialize panels
+
+	$tag_ref->{"knowledge_panels_" . $target_lc} = {};
+
+	my @panels = ();
+	if ($tagtype eq "categories") {
+		my $created
+			= create_category_packagings_materials_panel($tag_ref, $target_lc, $target_cc, $options_ref, $canon_tagid);
+		push(@panels, $created) if $created;
+	}
+	if (@panels) {
+		my $panel_data_ref = {tags_panels => \@panels};
+		# Create the root panel that contains the panels we want to show directly on the tag page
+		create_panel_from_json_template("root", "api/knowledge-panels/tags/root.tt.json",
+			$panel_data_ref, $tag_ref, $target_lc, $target_cc, $options_ref);
+	}
+
+	return !!@panels;
+}
+
+=head2 create_category_packagings_materials_panel ($tag_ref, $target_lc, $target_cc, $options_ref, $category_idf)
+
+Creates a knowledge panel to show the packagings materials stats of a category
+
+=head3 Arguments
+
+=head4 category id $category_id
+
+Canonical category id.
+
+=head4 tag reference $tag_ref
+
+Data about the tag, will store the knowledge panel.
+
+=head4 language code $target_lc
+
+Returned attributes contain both data and strings intended to be displayed to users.
+This parameter sets the desired language for the user facing strings.
+
+=head4 country code $target_cc
+
+=cut
+
+sub create_category_packagings_materials_panel ($tag_ref, $target_lc, $target_cc, $options_ref, $category_id) {
+
+	my $created;
+
+	my $categories_packagings_materials_stats_ref = load_categories_packagings_materials_stats();
+	my $country = canonicalize_taxonomy_tag("en", "countries", $target_cc);
+
+	$log->debug("create_category_packagings_materials_panel - start",
+		{target_lc => $target_lc, target_cc => $target_cc, country => $country, category_id => $category_id})
+		if $log->is_debug();
+
+	my $categories_ref
+		= deep_get($categories_packagings_materials_stats_ref, "countries", $country, "categories", $category_id);
+	if ($categories_ref) {
+		my $panel_data_ref = {
+			category_id => $category_id,
+			materials => $categories_ref->{materials},
+		};
+		create_panel_from_json_template("packagings_materials",
+			"api/knowledge-panels/tags/categories/packagings_materials.tt.json",
+			$panel_data_ref, $tag_ref, $target_lc, $target_cc, $options_ref);
+		$created = "packagings_materials";
+	}
+
+	$log->debug("create_category_packagings_materials_panel - end", {created => $created})
+		if $log->is_debug();
+
+	return $created;
+}
+
+1;

lib/ProductOpener/Packaging.pm

@@ -50,6 +50,8 @@ BEGIN {
 		&guess_language_of_packaging_text
 		&apply_rules_to_augment_packaging_component_data
 		&aggregate_packaging_by_parent_materials
+		&load_categories_packagings_materials_stats
+		&get_parent_material
 
 		%packaging_taxonomies
 	);    # symbols to export on request
@@ -70,6 +72,31 @@ use ProductOpener::ImportConvert qw/:all/;
 use Data::DeepAccess qw(deep_get deep_val);
 use List::Util qw(first);
 
+# We use a global variable in order to load the packaging stats only once
+my $categories_packagings_materials_stats_ref;
+
+sub load_categories_packagings_materials_stats() {
+	if (not defined $categories_packagings_materials_stats_ref) {
+		my $file = "$data_root/data/categories_stats/categories_packagings_materials_stats.all.popular.json";
+		# In dev environments, we provide a sample stats file in the data-default directory
+		# so that we can run tests with meaningful and unchanging data
+		if (!-e $file) {
+			my $default_file
+				= "$data_root/data-default/categories_stats/categories_packagings_materials_stats.all.popular.json";
+			$log->debug("local packaging stats file does not exist, will use default",
+				{file => $file, default_file => $default_file})
+				if $log->is_debug();
+			$file = $default_file;
+		}
+		$log->debug("loading packagings materials stats", {file => $file}) if $log->is_debug();
+		$categories_packagings_materials_stats_ref = retrieve_json($file);
+		if (not defined $categories_packagings_materials_stats_ref) {
+			$log->debug("unable to load packagings materials stats", {file => $file}) if $log->is_debug();
+		}
+	}
+	return $categories_packagings_materials_stats_ref;
+}
+
 =head1 FUNCTIONS
 
 =head2 extract_packagings_from_image( $product_ref $id $ocr_engine $results_ref )
@@ -852,6 +879,36 @@ sub set_packaging_misc_tags ($product_ref) {
 	return;
 }
 
+=head2 get_parent_material ($material)
+
+Return the parent material (glass, plastics, metal, paper or cardboard) of a material.
+Return unknown if the material does not match one of the parents, or if not defined.
+
+=cut
+
+# Build a cache of parent materials to speed up lookups
+my %parent_materials = ();
+
+sub get_parent_material ($material) {
+
+	return "en:unknown" if not defined $material;
+
+	# Check if we already computed the parent material
+	my $parent_material = $parent_materials{$material};
+	if (defined $parent_material) {
+		return $parent_material;
+	}
+	else {
+		# take first matching, most harmful first
+		$parent_material = (first {is_a("packaging_materials", $material, $_)}
+				("en:plastic", "en:glass", "en:metal", "en:paper-or-cardboard")) // "en:unknown";
+
+		$parent_materials{$material} = $parent_material;
+
+		return $parent_material;
+	}
+}
+
 =head2 aggregate_packaging_by_parent_materials ($product_ref)
 
 Aggregate the weights of each packaging component by parent material (glass, plastics, metal, paper or cardboard)
@@ -874,12 +931,7 @@ sub aggregate_packaging_by_parent_materials ($product_ref) {
 		foreach my $packaging_ref (@{$product_ref->{packagings}}) {
 
 			# Determine what is the parent material for the component
-			my $material = $packaging_ref->{material};
-			my $parent_material = "en:unknown";
-			if (defined $material) {
-				$parent_material = (first {is_a("packaging_materials", $material, $_)}
-						("en:paper-or-cardboard", "en:plastic", "en:glass", "en:metal")) // "en:unknown";
-			}
+			my $parent_material = get_parent_material($packaging_ref->{material});
 
 			# Initialize the entry for the parent material if needed (even if we have no weight,
 			# it is useful to know that there is some parent material used)