From 807fcf1fb7ac1803f8ca30bc51a3750c4d230f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Tue, 5 Sep 2023 16:14:40 +0200 Subject: [PATCH 1/9] feat: packagings support in search and graphs --- lib/ProductOpener/Display.pm | 211 ++++++++++++++++++----------------- 1 file changed, 106 insertions(+), 105 deletions(-) diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index 46aac7b8e75d1..82abfff593423 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -5802,6 +5802,35 @@ my %nutrition_grades_colors = ( unknown => {r => 128, g => 128, b => 128}, ); +# Return the path (list of nodes) to the search field + +# field name from the search form +# it can be: +# - a nutrient id like "saturated-fat" +# - a direct field like ingredients_n +# - an indirect field like packagings_materials.all.weight_100g + +sub get_search_field_path_components ($field) { + my @fields; + # direct fields + if ($field =~ /_n$/) { + @fields = ($field); + } + # indirect fields separated with the . character + elsif ($field =~ /\./) { + @fields = split(/\./, $field); + } + # forest footprint + elsif ($field eq "forest_footprint") { + @fields = ('forest_footprint_data', 'footprint_per_kg'); + } + # we assume other fields are nutrients ids + else { + @fields = ("nutriments", $field . "_100g"); + } + return @fields; +} + sub display_scatter_plot ($graph_ref, $products_ref) { my @products = @{$products_ref}; @@ -5826,7 +5855,7 @@ sub display_scatter_plot ($graph_ref, $products_ref) { $x_allowDecimals = "allowDecimals:true,\n"; $x_title = escape_single_quote(lang($graph_ref->{axis_x})); } - elsif ($graph_ref->{axis_x} =~ /ingredients_n/) { + elsif ($graph_ref->{axis_x} =~ /_n$/) { $x_allowDecimals = "allowDecimals:false,\n"; $x_title = escape_single_quote(lang($graph_ref->{axis_x} . "_s")); } @@ -5847,7 +5876,7 @@ sub display_scatter_plot ($graph_ref, $products_ref) { $y_allowDecimals = "allowDecimals:true,\n"; $y_title = escape_single_quote(lang($graph_ref->{axis_y})); } - elsif ($graph_ref->{axis_y} =~ /ingredients_n/) { + elsif ($graph_ref->{axis_y} =~ /_n$/) { $y_allowDecimals = "allowDecimals:false,\n"; $y_title = escape_single_quote(lang($graph_ref->{axis_y} . "_s")); } @@ -5871,118 +5900,96 @@ sub display_scatter_plot ($graph_ref, $products_ref) { foreach my $product_ref (@products) { + # Gather the data for the 2 axis + + my %data; + + foreach my $axis ('x', 'y') { + + my $field = $graph_ref->{"axis_" . $axis}; + my @fields = get_search_field_path_components($field); + my $value = deep_get($product_ref, @fields); + + # For nutrients except energy-kcal, convert to grams + if ((defined $value) and ($fields[0] eq "nutriments") and ($fields[1] ne "energy-kcal")) { + $value = g_to_unit($product_ref->{nutriments}{"${field}_100g"}, + (get_property("nutrients", "zz:$field", "unit:en") // 'g')); + } + + $data{$axis} = $value; + } + # Keep only products that have known values for both x and y + next if (not defined $data{x} or (not defined $data{y})); - if ( - ( - ( - (($graph_ref->{axis_x} eq 'additives_n') or ($graph_ref->{axis_x} =~ /ingredients_n$/)) - and (defined $product_ref->{$graph_ref->{axis_x}}) - ) - or (($graph_ref->{axis_x} eq 'forest_footprint') and (defined $product_ref->{forest_footprint_data})) - or (defined $product_ref->{nutriments}{$graph_ref->{axis_x} . "_100g"}) - and ($product_ref->{nutriments}{$graph_ref->{axis_x} . "_100g"} ne '') - ) - and ( - ( - (($graph_ref->{axis_y} eq 'additives_n') or ($graph_ref->{axis_y} =~ /ingredients_n$/)) - and (defined $product_ref->{$graph_ref->{axis_y}}) - ) - or (($graph_ref->{axis_y} eq 'forest_footprint') and (defined $product_ref->{forest_footprint_data})) - or (defined $product_ref->{nutriments}{$graph_ref->{axis_y} . "_100g"}) - and ($product_ref->{nutriments}{$graph_ref->{axis_y} . "_100g"} ne '') - ) - ) - { + # Add values to stats, and set min axis + foreach my $axis ('x', 'y') { + my $field = $graph_ref->{"axis_" . $axis}; + add_product_nutriment_to_stats(\%nutriments, $field, $data{$axis}); - my $url = $formatted_subdomain . product_url($product_ref->{code}); + # Set the minimum value for the axis (0 in most cases, except for Nutri-Score) + $min{$axis} = 0; - # Identify the series id - my $seriesid = 0; - my $s = 1000000; + if ($field =~ /^nutrition-score/) { + $min{$axis} = -15; + } - # default, organic, fairtrade, with_sweeteners - # order: organic, organic+fairtrade, organic+fairtrade+sweeteners, organic+sweeteners, fairtrade, fairtrade + sweeteners - # + } - # Colors for nutrition grades - if ($graph_ref->{"series_nutrition_grades"}) { - if (defined $product_ref->{"nutrition_grade_fr"}) { - $seriesid = $product_ref->{"nutrition_grade_fr"}; - } - else { - $seriesid = 'unknown'; - } + # Identify the series id + my $seriesid = 0; + my $s = 1000000; + + # default, organic, fairtrade, with_sweeteners + # order: organic, organic+fairtrade, organic+fairtrade+sweeteners, organic+sweeteners, fairtrade, fairtrade + sweeteners + # + + # Colors for nutrition grades + if ($graph_ref->{"series_nutrition_grades"}) { + if (defined $product_ref->{"nutrition_grade_fr"}) { + $seriesid = $product_ref->{"nutrition_grade_fr"}; } else { - # Colors for labels and labels combinations - foreach my $series (@search_series) { - # Label? - if ($graph_ref->{"series_$series"}) { - if (defined lang("search_series_${series}_label")) { - if (has_tag($product_ref, "labels", 'en:' . lc($Lang{"search_series_${series}_label"}{en}))) - { - $seriesid += $s; - } - else { - } - } - - if ($product_ref->{$series}) { + $seriesid = 'unknown'; + } + } + else { + # Colors for labels and labels combinations + foreach my $series (@search_series) { + # Label? + if ($graph_ref->{"series_$series"}) { + if (defined lang("search_series_${series}_label")) { + if (has_tag($product_ref, "labels", 'en:' . lc($Lang{"search_series_${series}_label"}{en}))) { $seriesid += $s; } + else { + } } - if (($series eq 'default') and ($seriesid == 0)) { + if ($product_ref->{$series}) { $seriesid += $s; } - $s = $s / 10; } - } - defined $series{$seriesid} or $series{$seriesid} = ''; - - # print STDERR "Display::search_and_graph_products: i: $i - axis_x: $graph_ref->{axis_x} - axis_y: $graph_ref->{axis_y}\n"; - - my %data; - - foreach my $axis ('x', 'y') { - my $nid = $graph_ref->{"axis_" . $axis}; + if (($series eq 'default') and ($seriesid == 0)) { + $seriesid += $s; + } + $s = $s / 10; + } + } - $min{$axis} = 0; + defined $series{$seriesid} or $series{$seriesid} = ''; - # number of ingredients, additives etc. (ingredients_n) - if ($nid =~ /_n$/) { - $data{$axis} = $product_ref->{$nid}; - } - elsif ($nid eq "forest_footprint") { - $data{$axis} = $product_ref->{forest_footprint_data}{footprint_per_kg}; - } - # energy-kcal is already in kcal - elsif ($nid eq 'energy-kcal') { - $data{$axis} = $product_ref->{nutriments}{"${nid}_100g"}; - } - elsif ($nid =~ /^nutrition-score/) { - $data{$axis} = $product_ref->{nutriments}{"${nid}_100g"}; - $min{$axis} = -15; - } - else { - $data{$axis} = g_to_unit($product_ref->{nutriments}{"${nid}_100g"}, - (get_property("nutrients", "zz:$nid", "unit:en") // 'g')); - } + $data{product_name} = $product_ref->{product_name}; + $data{url} = $formatted_subdomain . product_url($product_ref->{code}); + $data{img} = display_image_thumb($product_ref, 'front'); - add_product_nutriment_to_stats(\%nutriments, $nid, $product_ref->{nutriments}{"${nid}_100g"}); - } - $data{product_name} = $product_ref->{product_name}; - $data{url} = $url; - $data{img} = display_image_thumb($product_ref, 'front'); + defined $series{$seriesid} or $series{$seriesid} = ''; + $series{$seriesid} .= JSON::PP->new->encode(\%data) . ','; + defined $series_n{$seriesid} or $series_n{$seriesid} = 0; + $series_n{$seriesid}++; + $i++; - defined $series{$seriesid} or $series{$seriesid} = ''; - $series{$seriesid} .= JSON::PP->new->encode(\%data) . ','; - defined $series_n{$seriesid} or $series_n{$seriesid} = 0; - $series_n{$seriesid}++; - $i++; - } } my $series_data = ''; @@ -6608,17 +6615,11 @@ sub search_and_graph_products ($request_ref, $query_ref, $graph_ref) { } foreach my $axis ('x', 'y') { - if ($graph_ref->{"axis_$axis"} ne "products_n") { - if ($graph_ref->{"axis_$axis"} eq "forest_footprint") { - $fields_ref->{"forest_footprint_data.footprint_per_kg"} = 1; - } - elsif ($graph_ref->{"axis_$axis"} =~ /_n$/) { - $fields_ref->{$graph_ref->{"axis_$axis"}} = 1; - } - else { - $fields_ref->{"nutriments." . $graph_ref->{"axis_$axis"} . "_100g"} = 1; - } - } + my @field = $graph_ref->{"axis_$axis"}; + # Get the field path components + my @fields = get_search_field_path_components($field); + # Convert to dot notation to get the MongoDB field + $fields_ref->{join(".", @fields)} = 1; } if ($graph_ref->{"series_nutrition_grades"}) { From 137e4dccbf197cf9b6b8bf7e1e37d15f7718d29b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Tue, 5 Sep 2023 16:45:31 +0200 Subject: [PATCH 2/9] search packagings --- lib/ProductOpener/Display.pm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index 82abfff593423..48800180876b8 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -76,6 +76,7 @@ BEGIN { &display_product_history &display_preferences_api &display_attribute_groups_api + &get_search_field_path_components &search_and_display_products &search_and_export_products &search_and_graph_products @@ -6614,8 +6615,9 @@ sub search_and_graph_products ($request_ref, $query_ref, $graph_ref) { } } + # Add fields for the axis foreach my $axis ('x', 'y') { - my @field = $graph_ref->{"axis_$axis"}; + my $field = $graph_ref->{"axis_$axis"}; # Get the field path components my @fields = get_search_field_path_components($field); # Convert to dot notation to get the MongoDB field From 042297ac779845a813706dd463a7e7bb14eab5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Tue, 5 Sep 2023 19:04:05 +0200 Subject: [PATCH 3/9] search refactor --- cgi/search.pl | 14 ++-- lib/ProductOpener/Display.pm | 131 +++++++++++++++++------------------ po/common/common.pot | 11 +++ po/common/en.po | 11 +++ 4 files changed, 92 insertions(+), 75 deletions(-) diff --git a/cgi/search.pl b/cgi/search.pl index 3d3294167add9..a890d10c63179 100755 --- a/cgi/search.pl +++ b/cgi/search.pl @@ -738,13 +738,13 @@ # We want existing values for axis fields foreach my $axis ('x', 'y') { - if ( ($graph_ref->{"axis_$axis"} ne "") - and ($graph_ref->{"axis_$axis"} ne "forest_footprint") - and ($graph_ref->{"axis_$axis"} !~ /_n$/)) - { - (defined $query_ref->{"nutriments." . $graph_ref->{"axis_$axis"} . "_100g"}) - or $query_ref->{"nutriments." . $graph_ref->{"axis_$axis"} . "_100g"} = {}; - $query_ref->{"nutriments." . $graph_ref->{"axis_$axis"} . "_100g"}{'$exists'} = 1; + + if ($graph_ref->{"axis_$axis"} ne "") { + my $field = $graph_ref->{"axis_$axis"}; + # Get the field path components + my @fields = get_search_field_path_components($field); + # Convert to dot notation to get the MongoDB field + $query_ref->{join(".", @fields)} = {'$exists' => 1}; } } diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index 48800180876b8..fbfb8f815008c 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -5832,63 +5832,65 @@ sub get_search_field_path_components ($field) { return @fields; } -sub display_scatter_plot ($graph_ref, $products_ref) { - - my @products = @{$products_ref}; - my $count = @products; +sub get_search_field_title_and_details ($field) { - my $html = ''; + my ($title, $unit, $unit2, $allow_decimals) = ('', '', '', ''); - my $x_allowDecimals = ''; - my $y_allowDecimals = ''; - my $x_title; - my $y_title; - my $x_unit = ''; - my $y_unit = ''; - my $x_unit2 = ''; - my $y_unit2 = ''; - - if ($graph_ref->{axis_x} eq 'additives_n') { - $x_allowDecimals = "allowDecimals:false,\n"; - $x_title = escape_single_quote(lang("number_of_additives")); - } - elsif ($graph_ref->{axis_x} eq "forest_footprint") { - $x_allowDecimals = "allowDecimals:true,\n"; - $x_title = escape_single_quote(lang($graph_ref->{axis_x})); + if ($field eq 'additives_n') { + $allow_decimals = "allowDecimals:false,\n"; + $title = escape_single_quote(lang("number_of_additives")); } - elsif ($graph_ref->{axis_x} =~ /_n$/) { - $x_allowDecimals = "allowDecimals:false,\n"; - $x_title = escape_single_quote(lang($graph_ref->{axis_x} . "_s")); - } - else { - $x_title = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_x}); - $x_unit - = " (" - . (get_property("nutrients", "zz:" . $graph_ref->{axis_x}, "unit:en") // 'g') . " " - . lang("nutrition_data_per_100g") . ")"; - $x_unit =~ s/\ / /g; - $x_unit2 = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_x}); - } - if ($graph_ref->{axis_y} eq 'additives_n') { - $y_allowDecimals = "allowDecimals:false,\n"; - $y_title = escape_single_quote(lang("number_of_additives")); + elsif ($field eq "forest_footprint") { + $allow_decimals = "allowDecimals:true,\n"; + $title = escape_single_quote(lang($field)); } - elsif ($graph_ref->{axis_y} eq "forest_footprint") { - $y_allowDecimals = "allowDecimals:true,\n"; - $y_title = escape_single_quote(lang($graph_ref->{axis_y})); + elsif ($field =~ /_n$/) { + $allow_decimals = "allowDecimals:false,\n"; + $title = escape_single_quote(lang($field . "_s")); } - elsif ($graph_ref->{axis_y} =~ /_n$/) { - $y_allowDecimals = "allowDecimals:false,\n"; - $y_title = escape_single_quote(lang($graph_ref->{axis_y} . "_s")); + elsif ($field =~ /^packagings_materials\.([^\.]+)\.([^\.]+)$/) { + $title = lang("packaging") . " - " . display_taxonomy_tag($lc, "packagings_materials", "zz:$1") . ' - ' . $2; } else { - $y_title = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_y}); - $y_unit + $title = display_taxonomy_tag($lc, "nutrients", "zz:" . $field); + $unit2 = $title; # displayed in the tooltip + $unit = " (" - . (get_property("nutrients", "zz:" . $graph_ref->{axis_y}, "unit:en") // 'g') . " " + . (get_property("nutrients", "zz:" . $field, "unit:en") // 'g') . " " . lang("nutrition_data_per_100g") . ")"; - $y_unit =~ s/\ / /g; - $y_unit2 = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_y}); + $unit =~ s/\ / /g; + } + + return ($title, $unit, $unit2, $allow_decimals); +} + +sub display_scatter_plot ($graph_ref, $products_ref) { + + my @products = @{$products_ref}; + my $count = @products; + + my $html = ''; + + my %axis_details = (); + my %min = (); # Minimum for the axis, 0 except -15 for Nutri-Score score + + foreach my $axis ("x", "y") { + # Set the titles and details of each axis + my $field = $graph_ref->{"axis_" . $axis}; + my ($title, $unit, $unit2, $allow_decimals) = get_search_field_title_and_details($field); + $axis_details{$axis} = { + title => $title, + unit => $unit, + unit2 => $unit2, + allow_decimals => $allow_decimals, + }; + + # Set the minimum value for the axis (0 in most cases, except for Nutri-Score) + $min{$axis} = 0; + + if ($field =~ /^nutrition-score/) { + $min{$axis} = -15; + } } my %nutriments = (); @@ -5897,7 +5899,6 @@ sub display_scatter_plot ($graph_ref, $products_ref) { my %series = (); my %series_n = (); - my %min = (); # Minimum for the axis, 0 except -15 for Nutri-Score score foreach my $product_ref (@products) { @@ -5911,30 +5912,24 @@ sub display_scatter_plot ($graph_ref, $products_ref) { my @fields = get_search_field_path_components($field); my $value = deep_get($product_ref, @fields); - # For nutrients except energy-kcal, convert to grams - if ((defined $value) and ($fields[0] eq "nutriments") and ($fields[1] ne "energy-kcal")) { - $value = g_to_unit($product_ref->{nutriments}{"${field}_100g"}, - (get_property("nutrients", "zz:$field", "unit:en") // 'g')); + # For nutrients except energy-kcal, convert to the default nutrient unit + if ((defined $value) and ($fields[0] eq "nutriments") and ($field !~ /energy-kcal/)) { + $value = g_to_unit($value, (get_property("nutrients", "zz:$field", "unit:en") // 'g')); } $data{$axis} = $value; + + $data{"a_field_$axis"} = $field; + $data{"a_value_$axis"} = $value; } # Keep only products that have known values for both x and y - next if (not defined $data{x} or (not defined $data{y})); + next if ((not defined $data{x}) or (not defined $data{y})); # Add values to stats, and set min axis foreach my $axis ('x', 'y') { my $field = $graph_ref->{"axis_" . $axis}; add_product_nutriment_to_stats(\%nutriments, $field, $data{$axis}); - - # Set the minimum value for the axis (0 in most cases, except for Nutri-Score) - $min{$axis} = 0; - - if ($field =~ /^nutrition-score/) { - $min{$axis} = -15; - } - } # Identify the series id @@ -6110,21 +6105,21 @@ JS text: '$Lang{data_source}{$lc}$sep: $formatted_subdomain' }, xAxis: { - $x_allowDecimals + $axis_details{x}{allow_decimals} min:$min{x}, title: { enabled: true, - text: '${x_title}${x_unit}' + text: '$axis_details{x}{title}$axis_details{x}{unit}' }, startOnTick: true, endOnTick: true, showLastLabel: true }, yAxis: { - $y_allowDecimals + $axis_details{y}{allow_decimals} min:$min{y}, title: { - text: '${y_title}${y_unit}' + text: '$axis_details{y}{title}$axis_details{y}{unit}' } }, tooltip: { @@ -6135,8 +6130,8 @@ JS return '' + this.point.product_name + '
' + this.point.img + '
' + '$Lang{nutrition_data_per_100g}{$lc} :' - + '
$x_title$sep: '+ this.x + ' $x_unit2' - + '
$y_title$sep: ' + this.y + ' $y_unit2'; + + '
$axis_details{x}{title}$sep: '+ this.x + ' $axis_details{x}{unit2}' + + '
$axis_details{y}{title}$sep: ' + this.y + ' $axis_details{y}{unit2}'; } }, diff --git a/po/common/common.pot b/po/common/common.pot index b8ea301eee0b4..ef8dcb632134d 100644 --- a/po/common/common.pot +++ b/po/common/common.pot @@ -6591,3 +6591,14 @@ msgctxt "relative_to_all_products" msgid "relative to all products" msgstr "relative to all products" +msgctxt "packagings_n_p" +msgid "Numbers of packaging components" +msgstr "Numbers of packaging components" + +msgctxt "packagings_n_s" +msgid "Number of packaging components" +msgstr "Number of packaging components" + +msgctxt "packagings_materials_all" +msgid "All materials" +msgstr "All materials" \ No newline at end of file diff --git a/po/common/en.po b/po/common/en.po index 85dcb121ee9c5..cbc2616fa87f7 100644 --- a/po/common/en.po +++ b/po/common/en.po @@ -6615,3 +6615,14 @@ msgctxt "relative_to_all_products" msgid "relative to all products" msgstr "relative to all products" +msgctxt "packagings_n_p" +msgid "Numbers of packaging components" +msgstr "Numbers of packaging components" + +msgctxt "packagings_n_s" +msgid "Number of packaging components" +msgstr "Number of packaging components" + +msgctxt "packagings_materials_all" +msgid "All materials" +msgstr "All materials" \ No newline at end of file From 6d76bedf421fadf4bf10a7925307a4387ad76c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 6 Sep 2023 12:48:31 +0200 Subject: [PATCH 4/9] packagings graphs --- lib/ProductOpener/Display.pm | 10 +++++++++- po/common/common.pot | 14 +++++++++++++- po/common/en.po | 14 +++++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index fbfb8f815008c..d851a31b6a57e 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -5849,7 +5849,15 @@ sub get_search_field_title_and_details ($field) { $title = escape_single_quote(lang($field . "_s")); } elsif ($field =~ /^packagings_materials\.([^\.]+)\.([^\.]+)$/) { - $title = lang("packaging") . " - " . display_taxonomy_tag($lc, "packagings_materials", "zz:$1") . ' - ' . $2; + $title = lang("packaging") . " - "; + if ($1 eq "all") { + $title .= lang("packagings_materials_all"); + } + else { + $title .= display_taxonomy_tag($lc, "packagings_materials", "zz:$1"); + } + $title .= ' - ' . lang($2); + display_taxonomy_tag($lc, "packagings_materials", "zz:$1") . ' - ' . $2; } else { $title = display_taxonomy_tag($lc, "nutrients", "zz:" . $field); diff --git a/po/common/common.pot b/po/common/common.pot index ef8dcb632134d..f192cf5de7cfe 100644 --- a/po/common/common.pot +++ b/po/common/common.pot @@ -6601,4 +6601,16 @@ msgstr "Number of packaging components" msgctxt "packagings_materials_all" msgid "All materials" -msgstr "All materials" \ No newline at end of file +msgstr "All materials" + +msgctxt "weight" +msgid "Weight" +msgstr "Weight" + +msgctxt "weight_100g" +msgid "Weight per 100g of product" +msgstr "Weight per 100g of product" + +msgctxt "weight_percent" +msgid "Weight percent" +msgstr "Weight percent" \ No newline at end of file diff --git a/po/common/en.po b/po/common/en.po index cbc2616fa87f7..fea79ee0e10b9 100644 --- a/po/common/en.po +++ b/po/common/en.po @@ -6625,4 +6625,16 @@ msgstr "Number of packaging components" msgctxt "packagings_materials_all" msgid "All materials" -msgstr "All materials" \ No newline at end of file +msgstr "All materials" + +msgctxt "weight" +msgid "Weight" +msgstr "Weight" + +msgctxt "weight_100g" +msgid "Weight per 100g of product" +msgstr "Weight per 100g of product" + +msgctxt "weight_percent" +msgid "Weight percent" +msgstr "Weight percent" \ No newline at end of file From 4053b647acea26700800191ea9add878b45c0f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 6 Sep 2023 15:34:41 +0200 Subject: [PATCH 5/9] search histogram refactor --- cgi/search.pl | 29 +++-- lib/ProductOpener/Display.pm | 231 +++++++++++++++++------------------ 2 files changed, 136 insertions(+), 124 deletions(-) diff --git a/cgi/search.pl b/cgi/search.pl index a890d10c63179..180c91bbe143c 100755 --- a/cgi/search.pl +++ b/cgi/search.pl @@ -322,16 +322,29 @@ $axis_labels{$nid} = display_taxonomy_tag($lc, "nutrients", "zz:$nid"); $log->debug("nutriments", {nid => $nid, value => $axis_labels{$nid}}) if $log->is_debug(); } - push @axis_values, "additives_n", "ingredients_n", "known_ingredients_n", "unknown_ingredients_n"; - push @axis_values, "fruits-vegetables-nuts-estimate-from-ingredients"; - push @axis_values, "forest_footprint"; - $axis_labels{additives_n} = lang("number_of_additives"); - $axis_labels{ingredients_n} = lang("ingredients_n_s"); - $axis_labels{known_ingredients_n} = lang("known_ingredients_n_s"); - $axis_labels{unknown_ingredients_n} = lang("unknown_ingredients_n_s"); + + my @other_search_fields = ( + "additives_n", "ingredients_n", + "known_ingredients_n", "unknown_ingredients_n", + "fruits-vegetables-nuts-estimate-from-ingredients", "forest_footprint", + "product_quantity", + ); + + # Add the fields related to packaging + foreach my $material ("all", "en:plastic", "en:glass", "en:metal", "en:paper-or-cardboard", "en:unknown") { + foreach my $subfield ("weight", "weight_100g", "weight_percent") { + push @other_search_fields, "packagings_materials.$material.$subfield"; + } + } + $axis_labels{search_nutriment} = lang("search_nutriment"); $axis_labels{products_n} = lang("number_of_products"); - $axis_labels{forest_footprint} = lang("forest_footprint"); + + foreach my $field (@other_search_fields) { + my ($title, $unit, $unit2, $allow_decimals) = get_search_field_title_and_details($field); + push @axis_values, $field; + $axis_labels{$field} = $title; + } my @sorted_axis_values = ("", sort({lc($axis_labels{$a}) cmp lc($axis_labels{$b})} @axis_values)); diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index d851a31b6a57e..014f2ea3bd5f8 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -77,6 +77,7 @@ BEGIN { &display_preferences_api &display_attribute_groups_api &get_search_field_path_components + &get_search_field_title_and_details &search_and_display_products &search_and_export_products &search_and_graph_products @@ -5814,7 +5815,7 @@ my %nutrition_grades_colors = ( sub get_search_field_path_components ($field) { my @fields; # direct fields - if ($field =~ /_n$/) { + if (($field =~ /_n$/) or ($field eq "product_quantity")) { @fields = ($field); } # indirect fields separated with the . character @@ -5848,16 +5849,35 @@ sub get_search_field_title_and_details ($field) { $allow_decimals = "allowDecimals:false,\n"; $title = escape_single_quote(lang($field . "_s")); } + elsif ($field eq "product_quantity") { + $allow_decimals = "allowDecimals:false,\n"; + $title = escape_single_quote(lang("quantity")); + $unit = ' (g)'; + $unit2 = 'g'; + } elsif ($field =~ /^packagings_materials\.([^\.]+)\.([^\.]+)$/) { + my $material = $1; + my $subfield = $2; $title = lang("packaging") . " - "; - if ($1 eq "all") { + if ($material eq "all") { $title .= lang("packagings_materials_all"); } else { - $title .= display_taxonomy_tag($lc, "packagings_materials", "zz:$1"); + $title .= display_taxonomy_tag($lc, "packaging_materials", $material); + } + $title .= ' - ' . lang($subfield); + if ($subfield =~ /_percent$/) { + $unit = ' %'; + $unit2 = '%'; + } + elsif ($subfield =~ /_100g$/) { + $unit = ' (g/100g)'; + $unit2 = 'g/100g'; + } + else { + $unit = ' (g)'; + $unit2 = 'g'; } - $title .= ' - ' . lang($2); - display_taxonomy_tag($lc, "packagings_materials", "zz:$1") . ' - ' . $2; } else { $title = display_taxonomy_tag($lc, "nutrients", "zz:" . $field); @@ -5881,6 +5901,7 @@ sub display_scatter_plot ($graph_ref, $products_ref) { my %axis_details = (); my %min = (); # Minimum for the axis, 0 except -15 for Nutri-Score score + my %fields = (); # fields path components for each axis, to use with deep_get() foreach my $axis ("x", "y") { # Set the titles and details of each axis @@ -5899,6 +5920,9 @@ sub display_scatter_plot ($graph_ref, $products_ref) { if ($field =~ /^nutrition-score/) { $min{$axis} = -15; } + + # Store the field path components + $fields{$field} = [get_search_field_path_components($field)]; } my %nutriments = (); @@ -5917,22 +5941,25 @@ sub display_scatter_plot ($graph_ref, $products_ref) { foreach my $axis ('x', 'y') { my $field = $graph_ref->{"axis_" . $axis}; - my @fields = get_search_field_path_components($field); - my $value = deep_get($product_ref, @fields); + my $value = deep_get($product_ref, @{$fields{$field}}); # For nutrients except energy-kcal, convert to the default nutrient unit - if ((defined $value) and ($fields[0] eq "nutriments") and ($field !~ /energy-kcal/)) { + if ((defined $value) and ($fields{$field}[0] eq "nutriments") and ($field !~ /energy-kcal/)) { $value = g_to_unit($value, (get_property("nutrients", "zz:$field", "unit:en") // 'g')); } - $data{$axis} = $value; + if (defined $value) { + $value = $value + 0; # Make sure the value is a number + } - $data{"a_field_$axis"} = $field; - $data{"a_value_$axis"} = $value; + $data{$axis} = $value; } # Keep only products that have known values for both x and y - next if ((not defined $data{x}) or (not defined $data{y})); + if ((not defined $data{x}) or (not defined $data{y})) { + $log->debug("Skipping product with unknown values ", {data => \%data}) if $log->is_debug(); + next; + } # Add values to stats, and set min axis foreach my $axis ('x', 'y') { @@ -6208,41 +6235,34 @@ sub display_histogram ($graph_ref, $products_ref) { my $html = ''; - my $x_allowDecimals = ''; - my $y_allowDecimals = ''; - my $x_title; - my $y_title; - my $x_unit = ''; - my $y_unit = ''; - my $x_unit2 = ''; - my $y_unit2 = ''; + my %axis_details = (); + my %min = (); # Minimum for the axis, 0 except -15 for Nutri-Score score - if ($graph_ref->{axis_x} eq 'additives_n') { - $x_allowDecimals = "allowDecimals:false,\n"; - $x_title = escape_single_quote(lang("number_of_additives")); - } - elsif ($graph_ref->{axis_x} eq "forest_footprint") { - $x_allowDecimals = "allowDecimals:true,\n"; - $x_title = escape_single_quote(lang($graph_ref->{axis_x})); - } - elsif ($graph_ref->{axis_x} =~ /ingredients_n$/) { - $x_allowDecimals = "allowDecimals:false,\n"; - $x_title = escape_single_quote(lang($graph_ref->{axis_x} . "_s")); - } - else { - $x_title = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_x}); - $x_unit - = " (" - . (get_property("nutrients", "zz:" . $graph_ref->{axis_x}, "unit:en") // 'g') . " " - . lang("nutrition_data_per_100g") . ")"; - $x_unit =~ s/\ / /g; - $x_unit2 = (get_property("nutrients", "zz:" . $graph_ref->{axis_x}, "unit:en") // 'g'); - } + foreach my $axis ("x") { + # Set the titles and details of each axis + my $field = $graph_ref->{"axis_" . $axis}; + my ($title, $unit, $unit2, $allow_decimals) = get_search_field_title_and_details($field); + $axis_details{$axis} = { + title => $title, + unit => $unit, + unit2 => $unit2, + allow_decimals => $allow_decimals, + }; - $y_allowDecimals = "allowDecimals:false,\n"; - $y_title = escape_single_quote(lang("number_of_products")); + # Set the minimum value for the axis (0 in most cases, except for Nutri-Score) + $min{$axis} = 0; - my $nid = $graph_ref->{"axis_x"}; + if ($field =~ /^nutrition-score/) { + $min{$axis} = -15; + } + } + + $axis_details{"y"} = { + title => escape_single_quote(lang("number_of_products")), + allow_decimals => "allowDecimals:false,\n", + unit => '', + unit2 => '', + }; my $i = 0; @@ -6253,92 +6273,71 @@ sub display_histogram ($graph_ref, $products_ref) { my $min = 10000000000000; my $max = -10000000000000; + my $field = $graph_ref->{"axis_x"}; + my @fields = get_search_field_path_components($field); + foreach my $product_ref (@products) { - # Keep only products that have known values for x + my $value = deep_get($product_ref, @fields); - if ( - ( - ( - (($graph_ref->{axis_x} eq 'additives_n') or ($graph_ref->{axis_x} =~ /ingredients_n$/)) - and (defined $product_ref->{$graph_ref->{axis_x}}) - ) - or (($graph_ref->{axis_x} eq 'forest_footprint') and (defined $product_ref->{forest_footprint_data})) - or (defined $product_ref->{nutriments}{$graph_ref->{axis_x} . "_100g"}) - and ($product_ref->{nutriments}{$graph_ref->{axis_x} . "_100g"} ne '') - ) - ) - { + # For nutrients except energy-kcal, convert to the default nutrient unit + if ((defined $value) and ($fields[0] eq "nutriments") and ($field !~ /energy-kcal/)) { + $value = g_to_unit($value, (get_property("nutrients", "zz:$field", "unit:en") // 'g')); + } - # Identify the series id - my $seriesid = 0; - my $s = 1000000; + # Keep only products that have known values for both x and y + if (not defined $value) { + next; + } - # default, organic, fairtrade, with_sweeteners - # order: organic, organic+fairtrade, organic+fairtrade+sweeteners, organic+sweeteners, fairtrade, fairtrade + sweeteners - # + $value = $value + 0; # Make sure the value is a number - foreach my $series (@search_series) { - # Label? - if ($graph_ref->{"series_$series"}) { - if (defined lang("search_series_${series}_label")) { - if (has_tag($product_ref, "labels", 'en:' . lc($Lang{"search_series_${series}_label"}{en}))) { - $seriesid += $s; - } - else { - } - } + if ($value < $min) { + $min = $value; + } + if ($value > $max) { + $max = $value; + } - if ($product_ref->{$series}) { + # Identify the series id + my $seriesid = 0; + my $s = 1000000; + + # default, organic, fairtrade, with_sweeteners + # order: organic, organic+fairtrade, organic+fairtrade+sweeteners, organic+sweeteners, fairtrade, fairtrade + sweeteners + # + + foreach my $series (@search_series) { + # Label? + if ($graph_ref->{"series_$series"}) { + if (defined lang("search_series_${series}_label")) { + if (has_tag($product_ref, "labels", 'en:' . lc($Lang{"search_series_${series}_label"}{en}))) { $seriesid += $s; } + else { + } } - if (($series eq 'default') and ($seriesid == 0)) { + if ($product_ref->{$series}) { $seriesid += $s; } - $s = $s / 10; } - # print STDERR "Display::search_and_graph_products: i: $i - axis_x: $graph_ref->{axis_x} - axis_y: $graph_ref->{axis_y}\n"; - - my $value = 0; - - # number of ingredients, additives etc. (ingredients_n) - if ($nid =~ /_n$/) { - $value = $product_ref->{$nid}; - } - elsif ($nid eq "forest_footprint") { - $value = $product_ref->{forest_footprint_data}{footprint_per_kg}; - } - # energy-kcal is already in kcal - elsif ($nid eq 'energy-kcal') { - $value = $product_ref->{nutriments}{"${nid}_100g"}; - } - elsif ($nid =~ /^nutrition-score/) { - $value = $product_ref->{nutriments}{"${nid}_100g"}; - } - else { - $value = g_to_unit($product_ref->{nutriments}{"${nid}_100g"}, - (get_property("nutrients", "zz:$nid", "unit:en") // 'g')); + if (($series eq 'default') and ($seriesid == 0)) { + $seriesid += $s; } + $s = $s / 10; + } - if ($value < $min) { - $min = $value; - } - if ($value > $max) { - $max = $value; - } + push @all_values, $value; - push @all_values, $value; + defined $series{$seriesid} or $series{$seriesid} = []; + push @{$series{$seriesid}}, $value; - defined $series{$seriesid} or $series{$seriesid} = []; - push @{$series{$seriesid}}, $value; + defined $series_n{$seriesid} or $series_n{$seriesid} = 0; + $series_n{$seriesid}++; + $i++; - defined $series_n{$seriesid} or $series_n{$seriesid} = 0; - $series_n{$seriesid}++; - $i++; - } } # define intervals @@ -6360,7 +6359,7 @@ sub display_histogram ($graph_ref, $products_ref) { push @intervals, [$min, $max, "$min"]; } else { - if (($nid =~ /_n$/) or ($nid =~ /^nutrition-score/)) { + if (($field =~ /_n$/) or ($field =~ /^nutrition-score/)) { $interval = 1; $intervals = 0; for (my $j = $min; $j <= $max; $j++) { @@ -6497,7 +6496,7 @@ JS xAxis: { title: { enabled: true, - text: '${x_title}${x_unit}' + text: '$axis_details{x}{title}$axis_details{x}{unit}' }, categories: [ $categories @@ -6505,10 +6504,10 @@ JS }, yAxis: { - $y_allowDecimals + $axis_details{y}{allow_decimals} min:0, title: { - text: '${y_title}${y_unit}' + text: '$axis_details{y}{title}' }, stackLabels: { enabled: true, @@ -6519,14 +6518,14 @@ JS } }, tooltip: { - headerFormat: '${x_title} {point.key}
${x_unit}', + headerFormat: '$axis_details{x}{title} {point.key}
$axis_details{x}{unit}
', pointFormat: '' + '', footerFormat: '
{series.name}: {point.y}
Total: {point.total}', shared: true, useHTML: true, formatter: function() { - var points=''; + var points='
${x_title} ' + this.x + '
${x_unit}
'; //loop each point in this.points \$.each(this.points,function(i,point){ points+='' From ddb05f7b06461d9ca4270f2938cca6383c923b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 6 Sep 2023 17:33:38 +0200 Subject: [PATCH 6/9] basic support for nova and ecoscore_score in search graphs --- cgi/search.pl | 12 +++++++----- lib/ProductOpener/Display.pm | 11 ++++++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cgi/search.pl b/cgi/search.pl index 180c91bbe143c..c393ab001e658 100755 --- a/cgi/search.pl +++ b/cgi/search.pl @@ -94,7 +94,8 @@ } my @search_fields - = qw(brands categories packaging labels origins manufacturing_places emb_codes purchase_places stores countries ingredients additives allergens traces nutrition_grades nova_groups languages creator editors states); + = qw(brands categories packaging labels origins manufacturing_places emb_codes purchase_places stores countries + ingredients additives allergens traces nutrition_grades nova_groups ecoscore languages creator editors states); $admin and push @search_fields, "lang"; @@ -109,6 +110,8 @@ allergens => 1, traces => 1, nutrition_grades => 1, + nova_groups => 1, + eco_score => 1, purchase_places => 1, stores => 1, countries => 1, @@ -324,10 +327,9 @@ } my @other_search_fields = ( - "additives_n", "ingredients_n", - "known_ingredients_n", "unknown_ingredients_n", - "fruits-vegetables-nuts-estimate-from-ingredients", "forest_footprint", - "product_quantity", + "additives_n", "ingredients_n", "known_ingredients_n", "unknown_ingredients_n", + "fruits-vegetables-nuts-estimate-from-ingredients", + "forest_footprint", "product_quantity", "nova_group", 'ecoscore_score', ); # Add the fields related to packaging diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index 014f2ea3bd5f8..ed11912e8f2ac 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -5815,7 +5815,8 @@ my %nutrition_grades_colors = ( sub get_search_field_path_components ($field) { my @fields; # direct fields - if (($field =~ /_n$/) or ($field eq "product_quantity")) { + if (($field =~ /_n$/) or ($field eq "product_quantity") or ($field eq "nova_group") or ($field eq "ecoscore_score")) + { @fields = ($field); } # indirect fields separated with the . character @@ -5855,6 +5856,14 @@ sub get_search_field_title_and_details ($field) { $unit = ' (g)'; $unit2 = 'g'; } + elsif ($field eq "nova_group") { + $allow_decimals = "allowDecimals:false,\n"; + $title = escape_single_quote(lang("nova_groups_s")); + } + elsif ($field eq "ecoscore_score") { + $allow_decimals = "allowDecimals:false,\n"; + $title = escape_single_quote(lang("ecoscore_score")); + } elsif ($field =~ /^packagings_materials\.([^\.]+)\.([^\.]+)$/) { my $material = $1; my $subfield = $2; From c6e3e2acdd1f97dfe731c08da7bada711404b06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 6 Sep 2023 17:50:37 +0200 Subject: [PATCH 7/9] Apply suggestions from code review Co-authored-by: Alex Garel --- lib/ProductOpener/Display.pm | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index ed11912e8f2ac..039a2568cb0e1 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -5864,7 +5864,7 @@ sub get_search_field_title_and_details ($field) { $allow_decimals = "allowDecimals:false,\n"; $title = escape_single_quote(lang("ecoscore_score")); } - elsif ($field =~ /^packagings_materials\.([^\.]+)\.([^\.]+)$/) { + elsif ($field =~ /^packagings_materials\.([^.]+)\.([^.]+)$/) { my $material = $1; my $subfield = $2; $title = lang("packaging") . " - "; @@ -5904,7 +5904,7 @@ sub get_search_field_title_and_details ($field) { sub display_scatter_plot ($graph_ref, $products_ref) { my @products = @{$products_ref}; - my $count = @products; + my $count = scalar @products; my $html = ''; @@ -6018,14 +6018,16 @@ sub display_scatter_plot ($graph_ref, $products_ref) { } } - defined $series{$seriesid} or $series{$seriesid} = ''; + $series{$seriesid} = $series{$seriesid} // ''; $data{product_name} = $product_ref->{product_name}; $data{url} = $formatted_subdomain . product_url($product_ref->{code}); $data{img} = display_image_thumb($product_ref, 'front'); + # create data entry for series defined $series{$seriesid} or $series{$seriesid} = ''; $series{$seriesid} .= JSON::PP->new->encode(\%data) . ','; + # count entries / series defined $series_n{$seriesid} or $series_n{$seriesid} = 0; $series_n{$seriesid}++; $i++; From 51545b558e99137e3bebecd6d08889121eb5cca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 6 Sep 2023 17:53:03 +0200 Subject: [PATCH 8/9] Update lib/ProductOpener/Display.pm Co-authored-by: Alex Garel --- lib/ProductOpener/Display.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index 039a2568cb0e1..8bed5d495e63f 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -5978,6 +5978,8 @@ sub display_scatter_plot ($graph_ref, $products_ref) { # Identify the series id my $seriesid = 0; + # series value, we start high for first series + # and second series value will have s / 10, etc. my $s = 1000000; # default, organic, fairtrade, with_sweeteners From 8551469d84a35d9f8c75e0728436287ca3927f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 6 Sep 2023 18:03:43 +0200 Subject: [PATCH 9/9] add some doc --- lib/ProductOpener/Display.pm | 40 ++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index 8bed5d495e63f..6bc6ed536fc2a 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -5901,6 +5901,22 @@ sub get_search_field_title_and_details ($field) { return ($title, $unit, $unit2, $allow_decimals); } +=head2 display_scatter_plot ($graph_ref, $products_ref) + +Called by search_and_graph_products() to display a scatter plot of products on 2 axis + +=head3 Arguments + +=head4 $graph_ref + +Options for the graph, set by /cgi/search.pl + +=head4 $products_ref + +List of search results from search_and_graph_products() + +=cut + sub display_scatter_plot ($graph_ref, $products_ref) { my @products = @{$products_ref}; @@ -5978,8 +5994,8 @@ sub display_scatter_plot ($graph_ref, $products_ref) { # Identify the series id my $seriesid = 0; - # series value, we start high for first series - # and second series value will have s / 10, etc. + # series value, we start high for first series + # and second series value will have s / 10, etc. my $s = 1000000; # default, organic, fairtrade, with_sweeteners @@ -6026,10 +6042,10 @@ sub display_scatter_plot ($graph_ref, $products_ref) { $data{url} = $formatted_subdomain . product_url($product_ref->{code}); $data{img} = display_image_thumb($product_ref, 'front'); - # create data entry for series + # create data entry for series defined $series{$seriesid} or $series{$seriesid} = ''; $series{$seriesid} .= JSON::PP->new->encode(\%data) . ','; - # count entries / series + # count entries / series defined $series_n{$seriesid} or $series_n{$seriesid} = 0; $series_n{$seriesid}++; $i++; @@ -6241,6 +6257,22 @@ HTML } +=head2 display_histogram ($graph_ref, $products_ref) + +Called by search_and_graph_products() to display an histogram of products on 1 axis + +=head3 Arguments + +=head4 $graph_ref + +Options for the graph, set by /cgi/search.pl + +=head4 $products_ref + +List of search results from search_and_graph_products() + +=cut + sub display_histogram ($graph_ref, $products_ref) { my @products = @{$products_ref};
$axis_details{x}{title} ' + this.x + '
$axis_details{x}{unit}
'+point.series.name+':