Skip to content

Commit

Permalink
Use new nutrients taxonomy to replace hardcoded values from Food.pm -…
Browse files Browse the repository at this point in the history 
… Ready for review (#6123)

* rename nutrients.txt to nutrients_old.txt

* rename nutrients.txt to nutrients_old.txt

* create script to generate nutrients taxonomy from Food.pm hardcoded values + nutrients_old.txt taxonomy

* update script

* rename iu / dv to iu_value / dv_value

* add xx: entry to all nutrients

* new nutrients taxonomy

* Small fixes to nutrients taxonomy

* use zz: entries for canonical nutrient taxonomy ids

* fix commas

* Start to use the nutrients taxonomy - #1748

* Start to use the nutrients taxonomy - #1748

* tests for PackagerCodes.pm

* add wikidata properties from nutrients_old.txt

* make canonicalize_nutriment() use the new nutrients taxonomy

* replacing %Nutriments by nutrients taxonomy

* fix typo

* fix compute_serving_size and tests

* fix nutrients suggestions

* fix serving size computation for nutrients

* fixes for units

* add missing added-sugars

* small fixes

* update test

* replace %Nutriment by nutrients taxonomy

* replace %Nutriment by nutrients taxonomy

* replace %Nutriment by nutrients taxonomy

* replace %Nutriment by nutrients taxonomy

* Remove %Nutriments from Food.pm as we now use the nutrients taxonomy

* fixed typo

* use new function from Tags.pm to get synonyms

* replace %Nutriment by nutrients taxonomy

* removing generate_nutrients_taxonomy.pl now that is has been used once

* update tests

* update tests

* fix test

* apply suggestions from code review

* remove old nutrients taxonomy
  • Loading branch information
@stephanegigandet
stephanegigandet authored Dec 3, 2021
1 parent cc77a2b commit 86d1247
Show file tree
Hide file tree
Showing 141 changed files with 9,935 additions and 10,646 deletions.
3 changes: 2 additions & 1 deletion cgi/nutrients.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
use ProductOpener::Lang qw/:all/;
use ProductOpener::Display qw/:all/;
use ProductOpener::Food qw/:all/;
use ProductOpener::Tags qw/:all/;

use Log::Any qw($log);
use CGI qw/:cgi :form escapeHTML/;
Expand Down Expand Up @@ -58,7 +59,7 @@
my $onid = $nid =~ s/^(\-+)//gr;

my $current_ref = { id => $onid, important => $important, display_in_edit_form => $default_edit_form };
my $name = get_nutrient_label($onid, $lc);
my $name = display_taxonomy_tag($lc, "nutrients", "zz:$onid");
if (defined $name) {
$current_ref->{name} = $name;
}
Expand Down
2 changes: 1 addition & 1 deletion cgi/product_jqm_multilingual.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,7 @@
my @unknown_nutriments = ();
foreach my $nid (sort keys %{$product_ref->{nutriments}}) {
next if $nid =~ /_/;
if ((not exists $Nutriments{$nid}) and (defined $product_ref->{nutriments}{$nid . "_label"})) {
if ((not exists_taxonomy_tag("nutrients", "zz:$nid")) and (defined $product_ref->{nutriments}{$nid . "_label"})) {
push @unknown_nutriments, $nid;
$log->debug("unknown nutrient", { nid => $nid }) if $log->is_debug();
}
Expand Down
75 changes: 31 additions & 44 deletions cgi/product_multilingual.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@

$nid =~ s/_prepared$//;

if ((not exists $Nutriments{$nid}) and (defined $product_ref->{nutriments}{$nid . "_label"})
if ((not exists_taxonomy_tag("nutrients", "zz:$nid")) and (defined $product_ref->{nutriments}{$nid . "_label"})
and (not defined $seen_unknown_nutriments{$nid})) {
push @unknown_nutriments, $nid;
$log->debug("unknown_nutriment", { nid => $nid }) if $log->is_debug();
Expand Down Expand Up @@ -1202,7 +1202,7 @@ ($$$$$)

$log->trace("detect unknown nutriment", { nid => $nid }) if $log->is_trace();

if ((not exists $Nutriments{$nid}) and (defined $product_ref->{nutriments}{$nid . "_label"})
if ((not exists_taxonomy_tag("nutrients", "zz:$nid")) and (defined $product_ref->{nutriments}{$nid . "_label"})
and (not defined $seen_unknown_nutriments{$nid})) {
push @unknown_nutriments, $nid;
$log->debug("unknown nutriment detected", { nid => $nid }) if $log->is_debug();
Expand Down Expand Up @@ -1254,30 +1254,23 @@ ($$$$$)
my $nidp = $nid . "_prepared";
my $enidp = encodeURIComponent($nidp);

my $label = '';
if ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{$lang})) {
$nutriment_ref->{nutriments_nid} = $Nutriments{$nid};
$nutriment_ref->{nutriments_nid_lang} = $Nutriments{$nid}{$lang};
}
elsif ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{en})) {
$nutriment_ref->{nutriments_nid} = $Nutriments{$nid};
$nutriment_ref->{nutriments_nid_en} = $Nutriments{$nid}{en};
}
elsif (defined $product_ref->{nutriments}{$nid . "_label"}) {
my $label_value = $product_ref->{nutriments}{$nid . "_label"};
}

$nutriment_ref->{label_value} = $product_ref->{nutriments}{$nid . "_label"};
$nutriment_ref->{product_add_nutrient} = $Lang{product_add_nutrient}{$lang};
$nutriment_ref->{prefix} = $prefix;

my $unit = 'g';
if ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{"unit_$cc"})) {
$unit = $Nutriments{$nid}{"unit_$cc"};
my $unit = "g";

if (exists_taxonomy_tag("nutrients", "zz:$nid")) {
$nutriment_ref->{name} = display_taxonomy_tag($lc, "nutrients", "zz:$nid");
# We may have a unit specific to the country (e.g. US nutrition facts table using the International Unit for this nutrient, and Europe using mg)
$unit = get_property("nutrients", "zz:$nid", "unit_$cc:en") // get_property("nutrients", "zz:$nid", "unit:en") // 'g';
}
elsif ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{unit})) {
$unit = $Nutriments{$nid}{unit};
else {
if (defined $product_ref->{nutriments}{$nid . "_unit"}) {
$unit = $product_ref->{nutriments}{$nid . "_unit"};
}
}

my $value; # product as sold
my $valuep; # prepared product

Expand Down Expand Up @@ -1353,12 +1346,12 @@ ($$$$$)
@units = ('mol/l', 'mmol/l', 'mval/l', 'ppm', "\N{U+00B0}rH", "\N{U+00B0}fH", "\N{U+00B0}e", "\N{U+00B0}dH", 'gpg');
}

if (((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{dv}) and ($Nutriments{$nid}{dv} > 0))
if ((defined get_property("nutrients", "zz:$nid", "dv:en"))
or ($nid =~ /^new_/)
or (uc($unit) eq '% DV')) {
push @units, '% DV';
}
if (((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{iu}) and ($Nutriments{$nid}{iu} > 0))
if ((defined get_property("nutrients", "zz:$nid", "iu:en"))
or ($nid =~ /^new_/)
or (uc($unit) eq 'IU')
or (uc($unit) eq 'UI')) {
Expand All @@ -1368,12 +1361,12 @@ ($$$$$)
my $hide_percent = '';
my $hide_select = '';

if ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{unit}) and ($Nutriments{$nid}{unit} eq '')) {
if ($unit eq '') {
$hide_percent = ' style="display:none"';
$hide_select = ' style="display:none"';

}
elsif ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{unit}) and ($Nutriments{$nid}{unit} eq '%')) {
elsif ($unit eq '%') {
$hide_select = ' style="display:none"';
}
else {
Expand Down Expand Up @@ -1412,7 +1405,6 @@ ($$$$$)
$nutriment_ref->{enid} = $enid;
$nutriment_ref->{enidp} = $enidp;
$nutriment_ref->{nid} = $nid;
$nutriment_ref->{label} = $label;
$nutriment_ref->{class} = $class;
$nutriment_ref->{value} = $value;
$nutriment_ref->{valuep} = $valuep;
Expand All @@ -1424,34 +1416,29 @@ ($$$$$)

$template_data_ref_display->{nutriments} = \@nutriments;

# Compute a list of nutrients that will not be displayed in the nutrition facts table in the product edit form
# because they are not set for the product, and are not displayed by default in the user's country.
# Users will be allowed to add those nutrients, and this list will be used for nutrient name autocompletion.

my $other_nutriments = '';
my $nutriments = '';
foreach my $nid (@{$other_nutriments_lists{$nutriment_table}}) {
my $other_nutriment_value;
if ((exists $Nutriments{$nid}{$lang}) and ($Nutriments{$nid}{$lang} ne '')) {
$other_nutriment_value = $Nutriments{$nid}{$lang};
}
else {
foreach my $olc (@{$country_languages{$cc}}, 'en') {
next if $olc eq $lang;
if ((exists $Nutriments{$nid}{$olc}) and ($Nutriments{$nid}{$olc} ne '')) {
$other_nutriment_value = $Nutriments{$nid}{$olc};
last;
}
}
}
my $other_nutriment_value = display_taxonomy_tag($lc, "nutrients", "zz:$nid");

if ((not (defined $other_nutriment_value)) or ($other_nutriment_value eq '')) {
$other_nutriment_value = $nid;
}
# Some nutrients cannot be entered directly by users, so don't suggest them
next if (get_property("nutrients", "zz:$nid", "automatically_computed:en") eq "yes");

if ((not defined $product_ref->{nutriments}{$nid}) or ($product_ref->{nutriments}{$nid} eq '')) {
my $supports_iu = "false";
if ((exists $Nutriments{$nid}{iu}) and ($Nutriments{$nid}{iu} > 0)) {
if (defined get_property("nutrients", "zz:$nid", "iu_value:en")) {
$supports_iu = "true";
}

$other_nutriments .= '{ "value" : "' . $other_nutriment_value . '", "unit" : "' . $Nutriments{$nid}{unit} . '", "iu": ' . $supports_iu . ' },' . "\n";

my $other_nutriment_unit = get_property("nutrients", "zz:$nid", "unit:en");
$other_nutriments .= '{ "value" : "' . $other_nutriment_value
. '", "unit" : "' . $other_nutriment_unit
. '", "iu": ' . $supports_iu
. ' },'. "\n";
}
$nutriments .= '"' . $other_nutriment_value . '" : "' . $nid . '",' . "\n";
}
Expand Down
4 changes: 2 additions & 2 deletions cgi/search.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
use ProductOpener::Products qw/:all/;
use ProductOpener::Food qw/:all/;
use ProductOpener::Tags qw/:all/;

use ProductOpener::PackagerCodes qw/:all/;

use CGI qw/:cgi :form escapeHTML/;
use URI::Escape::XS;
Expand Down Expand Up @@ -291,7 +291,7 @@
my @axis_values = @{$nutriments_lists{$nutriment_table}};
my %axis_labels = ();
foreach my $nid (@{$nutriments_lists{$nutriment_table}}, "fruits-vegetables-nuts-estimate-from-ingredients") {
$axis_labels{$nid} = ucfirst($Nutriments{$nid}{$lc} || $Nutriments{$nid}{en});
$axis_labels{$nid} = display_taxonomy_tag($lc, "nutrients", "zz:$nid");
$log->debug("nutriments", { nid => $nid, value => $axis_labels{$nid} }) if $log->is_debug();
}
push @axis_values, "additives_n", "ingredients_n", "known_ingredients_n", "unknown_ingredients_n";
Expand Down
1 change: 1 addition & 0 deletions cgi/suggest.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
use ProductOpener::Food qw/:all/;
use ProductOpener::Tags qw/:all/;
use ProductOpener::Lang qw/:all/;
use ProductOpener::PackagerCodes qw/:all/;

use CGI qw/:cgi :form escapeHTML/;
use URI::Escape::XS;
Expand Down
8 changes: 4 additions & 4 deletions lib/ProductOpener/Attributes.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,8 +355,8 @@ sub initialize_attribute($$) {
my $nid = $2;
$nid =~ s/_/-/g;

$attribute_ref->{name} = $Nutriments{$nid}{$target_lc};
$attribute_ref->{setting_name} = sprintf(lang_in_other_lc($target_lc, "nutrient_in_quantity"), $Nutriments{$nid}{$target_lc} ,
$attribute_ref->{name} = display_taxonomy_tag($target_lc, "nutrients", "zz:$nid");
$attribute_ref->{setting_name} = sprintf(lang_in_other_lc($target_lc, "nutrient_in_quantity"), display_taxonomy_tag($target_lc, "nutrients", "zz:$nid") ,
lang_in_other_lc($target_lc, $level . "_quantity"));
}

Expand Down Expand Up @@ -1070,7 +1070,7 @@ sub compute_attribute_nutrient_level($$$$) {
$attribute_ref->{status} = "unknown";
$attribute_ref->{icon_url} = "$static_subdomain/images/attributes/nutrient-level-$nid-unknown.svg";
if ($target_lc ne "data") {
$attribute_ref->{title} = sprintf(lang_in_other_lc($target_lc, "nutrient_in_quantity"), $Nutriments{$nid}{$target_lc} ,
$attribute_ref->{title} = sprintf(lang_in_other_lc($target_lc, "nutrient_in_quantity"), display_taxonomy_tag($target_lc, "nutrients", "zz:$nid") ,
lang_in_other_lc($target_lc, "unknown_quantity"));
$attribute_ref->{missing} = lang_in_other_lc($target_lc, "missing_nutrition_facts");
}
Expand Down Expand Up @@ -1120,7 +1120,7 @@ sub compute_attribute_nutrient_level($$$$) {
$attribute_ref->{match} = $match;

if ($target_lc ne "data") {
$attribute_ref->{title} = sprintf(lang_in_other_lc($target_lc, "nutrient_in_quantity"), $Nutriments{$nid}{$target_lc} ,
$attribute_ref->{title} = sprintf(lang_in_other_lc($target_lc, "nutrient_in_quantity"), display_taxonomy_tag($target_lc, "nutrients", "zz:$nid"),
lang_in_other_lc($target_lc, $product_ref->{nutrient_levels}{$nid} . "_quantity"));
$attribute_ref->{description_short} = sprintf(
lang_in_other_lc($target_lc, 'g_per_100g'),
Expand Down
2 changes: 1 addition & 1 deletion lib/ProductOpener/Config_off.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ vitamins minerals amino_acids nucleotides other_nutritional_substances allergens
nutrient_levels misc ingredients ingredients_analysis nova_groups ingredients_processing
data_quality data_quality_bugs data_quality_info data_quality_warnings data_quality_errors data_quality_warnings_producers data_quality_errors_producers
improvements origins packaging_shapes packaging_materials packaging_recycling
periods_after_opening
periods_after_opening nutrients
);


Expand Down
59 changes: 26 additions & 33 deletions lib/ProductOpener/Display.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ use ProductOpener::KnowledgePanels qw(:all);
use ProductOpener::Orgs qw(:all);
use ProductOpener::Web qw(:all);
use ProductOpener::Recipes qw(:all);
use ProductOpener::PackagerCodes qw(:all);

use Cache::Memcached::Fast;
use Encode;
Expand Down Expand Up @@ -5948,10 +5949,10 @@ sub display_scatter_plot($$) {
$x_title = escape_single_quote(lang($graph_ref->{axis_x} . "_s"));
}
else {
$x_title = $Nutriments{$graph_ref->{axis_x}}{$lc};
$x_unit = " (" . $Nutriments{$graph_ref->{axis_x}}{unit} . " " . lang("nutrition_data_per_100g") . ")";
$x_title = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_x});
$x_unit = " (" . (get_property("nutrients", "zz:" . $graph_ref->{axis_x}, "unit:en") // 'g') . " " . lang("nutrition_data_per_100g") . ")";
$x_unit =~ s/\ / /g;
$x_unit2 = $Nutriments{$graph_ref->{axis_x}}{unit};
$x_unit2 = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_x});
}
if ($graph_ref->{axis_y} eq 'additives_n') {
$y_allowDecimals = "allowDecimals:false,\n";
Expand All @@ -5966,10 +5967,10 @@ sub display_scatter_plot($$) {
$y_title = escape_single_quote(lang($graph_ref->{axis_y} . "_s"));
}
else {
$y_title = $Nutriments{$graph_ref->{axis_y}}{$lc};
$y_unit = " (" . $Nutriments{$graph_ref->{axis_y}}{unit} . " " . lang("nutrition_data_per_100g") . ")";
$y_title = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_y});
$y_unit = " (" . (get_property("nutrients", "zz:" . $graph_ref->{axis_y}, "unit:en") // 'g') . " " . lang("nutrition_data_per_100g") . ")";
$y_unit =~ s/\ / /g;
$y_unit2 = $Nutriments{$graph_ref->{axis_y}}{unit};
$y_unit2 = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_y});
}

my %nutriments = ();
Expand Down Expand Up @@ -6066,7 +6067,7 @@ sub display_scatter_plot($$) {
$min{$axis} = -15;
}
else {
$data{$axis} = g_to_unit($product_ref->{nutriments}{"${nid}_100g"}, $Nutriments{$nid}{unit});
$data{$axis} = g_to_unit($product_ref->{nutriments}{"${nid}_100g"}, (get_property("nutrients", "zz:$nid", "unit:en") // 'g') );
}

add_product_nutriment_to_stats(\%nutriments, $nid, $product_ref->{nutriments}{"${nid}_100g"});
Expand Down Expand Up @@ -6313,10 +6314,10 @@ sub display_histogram($$) {
$x_title = escape_single_quote(lang($graph_ref->{axis_x} . "_s"));
}
else {
$x_title = $Nutriments{$graph_ref->{axis_x}}{$lc};
$x_unit = " (" . $Nutriments{$graph_ref->{axis_x}}{unit} . " " . lang("nutrition_data_per_100g") . ")";
$x_title = display_taxonomy_tag($lc, "nutrients", "zz:" . $graph_ref->{axis_x});
$x_unit = " (" . (get_property("nutrients", "zz:" . $graph_ref->{axis_x}, "unit:en") // 'g') . " " . lang("nutrition_data_per_100g") . ")";
$x_unit =~ s/\ / /g;
$x_unit2 = $Nutriments{$graph_ref->{axis_x}}{unit};
$x_unit2 = (get_property("nutrients", "zz:" . $graph_ref->{axis_x}, "unit:en") // 'g');
}

$y_allowDecimals = "allowDecimals:false,\n";
Expand Down Expand Up @@ -6394,7 +6395,7 @@ sub display_histogram($$) {
$value = $product_ref->{nutriments}{"${nid}_100g"};
}
else {
$value = g_to_unit($product_ref->{nutriments}{"${nid}_100g"}, $Nutriments{$nid}{unit});
$value = g_to_unit($product_ref->{nutriments}{"${nid}_100g"}, (get_property("nutrients", "zz:$nid", "unit:en") // 'g'));
}

if ($value < $min) {
Expand Down Expand Up @@ -9162,7 +9163,7 @@ sub data_to_display_nutriscore_and_nutrient_levels($) {
push @{$result_data_ref->{nutrient_levels}}, {
nutrient_level => $product_ref->{nutrient_levels}{$nid},
nutriment_prepared => sprintf("%.2e", $product_ref->{nutriments}{$nid . $prepared . "_100g"}) + 0.0,
nutriment_quantity => sprintf(lang("nutrient_in_quantity"), "<b>" . $Nutriments{$nid}{$lc} . "</b>", lang($product_ref->{nutrient_levels}{$nid} . "_quantity")),
nutriment_quantity => sprintf(lang("nutrient_in_quantity"), "<b>" . display_taxonomy_tag($lc, "nutrients", "zz:$nid") . "</b>", lang($product_ref->{nutrient_levels}{$nid} . "_quantity")),
};
}
}
Expand Down Expand Up @@ -9560,7 +9561,7 @@ CSS

$nid =~ s/_prepared$//;

if ((not exists $Nutriments{$nid}) and (defined $product_ref->{nutriments}{$nid . "_label"})
if ((not exists_taxonomy_tag("nutrients", "zz:$nid")) and (defined $product_ref->{nutriments}{$nid . "_label"})
and (not defined $seen_unknown_nutriments{$nid})) {
push @unknown_nutriments, $nid;
$seen_unknown_nutriments{$nid} = 1;
Expand Down Expand Up @@ -9634,27 +9635,19 @@ CSS
# Name of the nutrient

my $name;
my $unit = "g";

if ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{$lang})) {
$name = $Nutriments{$nid}{$lang};

}
elsif ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{en})) {
$name = $Nutriments{$nid}{en};

if (exists_taxonomy_tag("nutrients", "zz:$nid")) {
$name = display_taxonomy_tag($lc, "nutrients", "zz:$nid");
$unit = get_property("nutrients", "zz:$nid", "unit:en") // 'g';
}
elsif (defined $product_ref->{nutriments}{$nid . "_label"}) {
$name = $product_ref->{nutriments}{$nid . "_label"};
}

my $unit = 'g';

if ((exists $Nutriments{$nid}) and (exists $Nutriments{$nid}{unit})) {
$unit = $Nutriments{$nid}{unit};

}
elsif ((not exists $Nutriments{$nid}) and (defined $product_ref->{nutriments}{$nid . "_unit"})) {
$unit = $product_ref->{nutriments}{$nid . "_unit"};
else {
if (defined $product_ref->{nutriments}{$nid . "_label"}) {
$name = $product_ref->{nutriments}{$nid . "_label"};
}
if (defined $product_ref->{nutriments}{$nid . "_unit"}) {
$unit = $product_ref->{nutriments}{$nid . "_unit"};
}
}

my $values;
Expand Down Expand Up @@ -9947,7 +9940,7 @@ CSS
if (($nid eq 'salt') and ($values2 ne '')) {

push @{$template_data_ref->{nutrition_table}{rows}}, {
name => $Nutriments{sodium}{$lang},
name => display_taxonomy_tag($lc, "nutrients", "zz:sodium"),
nid => "sodium",
level => 1,
columns => \@extra_row_columns,
Expand Down
Loading

0 comments on commit 86d1247

Please sign in to comment.