Skip to content

Commit

Permalink
feat: Estimate Ingredient percentages from nutritional data (#8025)
Browse files Browse the repository at this point in the history 
* Remove suppression of Eco-Score but add a warning

* Test data changes (not all seem linked to my changes)

* Display a short warning in the attribute panel

* Updated test results

* Perltidy fixes

* Forgot to build_lang_test

* Add the CIQUAL code into ingredients so we can see how many have full data already

* Export sub and store list of unmatched ingredients

* Revert changes that should not have been included in this PR

* Remooving other changes not in this PR

* More test results. Tests not passing locally

* Re-introduce changes deleted in error

* More unintended changes

* Delete rather than set to undef

* Updating test results

* Coerce number
  • Loading branch information
@john-gom
john-gom authored May 15, 2023
1 parent dbf1da6 commit c1ebb27
Show file tree
Hide file tree
Showing 142 changed files with 1,023 additions and 1 deletion.
32 changes: 32 additions & 0 deletions lib/ProductOpener/Ingredients.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ BEGIN {
&match_ingredient_origin
&parse_origins_from_text
&assign_ciqual_codes
); # symbols to export on request
%EXPORT_TAGS = (all => [@EXPORT_OK]);
}
Expand Down Expand Up @@ -2372,6 +2373,9 @@ sub extract_ingredients_from_text ($product_ref) {
# Add properties like origins from specific ingredients extracted from labels or the end of the ingredients list
add_properties_from_specific_ingredients($product_ref);

# Obtain Ciqual codes ready for ingredients estimation from nutrients
assign_ciqual_codes($product_ref);

# Compute minimum and maximum percent ranges for each ingredient and sub ingredient

if (compute_ingredients_percent_values(100, 100, $product_ref->{ingredients}) < 0) {
Expand Down Expand Up @@ -2408,6 +2412,34 @@ sub extract_ingredients_from_text ($product_ref) {
return;
}

sub assign_ciqual_codes ($product_ref) {
my @ingredients_without_ciqual_codes = uniq(sort(get_missing_ciqual_codes($product_ref->{ingredients})));
$product_ref->{ingredients_without_ciqual_codes} = \@ingredients_without_ciqual_codes;
$product_ref->{ingredients_without_ciqual_codes_n} = @ingredients_without_ciqual_codes + 0.0;
return;
}

sub get_missing_ciqual_codes ($ingredients_ref) {
my @ingredients_without_ciqual_codes = ();
foreach my $ingredient_ref (@{$ingredients_ref}) {
if (defined $ingredient_ref->{ingredients}) {
push(@ingredients_without_ciqual_codes, get_missing_ciqual_codes($ingredient_ref->{ingredients}));
}
else {
my $ciqual_food_code = get_inherited_property("ingredients", $ingredient_ref->{id}, "ciqual_food_code:en");
if (defined $ciqual_food_code) {
$ingredient_ref->{ciqual_food_code} = $ciqual_food_code;
}
else {
exists $ingredient_ref->{ciqual_food_code} and delete $ingredient_ref->{ciqual_food_code};
push(@ingredients_without_ciqual_codes, $ingredient_ref->{id});
}
}
}

return @ingredients_without_ciqual_codes;
}

=head2 delete_ingredients_percent_values ( ingredients_ref )
This function deletes the percent_min and percent_max values of all ingredients.
Expand Down
9 changes: 8 additions & 1 deletion scripts/update_all_products.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@
my $compute_main_countries = '';
my $prefix_packaging_tags_with_language = '';
my $fix_non_string_ids = '';
my $assign_ciqual_codes = '';

my $query_ref = {}; # filters for mongodb query

Expand Down Expand Up @@ -188,6 +189,7 @@
"fix-nutrition-data" => \$fix_nutrition_data,
"compute-main-countries" => \$compute_main_countries,
"prefix-packaging-tags-with-language" => \$prefix_packaging_tags_with_language,
"assign-ciqual-codes" => \$assign_ciqual_codes,
) or die("Error in command line arguments:\n\n$usage");

use Data::Dumper;
Expand Down Expand Up @@ -258,7 +260,8 @@
and (scalar @fields_to_update == 0)
and (not $count)
and (not $just_print_codes)
and (not $prefix_packaging_tags_with_language))
and (not $prefix_packaging_tags_with_language)
and (not $assign_ciqual_codes))
{
die("Missing fields to update or --count option:\n$usage");
}
Expand Down Expand Up @@ -1311,6 +1314,10 @@
}
}

if ($assign_ciqual_codes) {
assign_ciqual_codes($product_ref);
}

if (not $pretend) {
$product_ref->{update_key} = $key;

Expand Down
7 changes: 7 additions & 0 deletions tests/integration/expected_test_results/api_v2_product_read/get-existing-product.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -596,6 +597,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -605,6 +607,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -676,6 +679,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"known_ingredients_n" : 9,
Expand Down
7 changes: 7 additions & 0 deletions ...ntegration/expected_test_results/api_v2_product_read/get-fields-all-knowledge-panels.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -596,6 +597,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -605,6 +607,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -676,6 +679,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"knowledge_panels" : {
Expand Down
7 changes: 7 additions & 0 deletions tests/integration/expected_test_results/api_v2_product_read/get-fields-all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -596,6 +597,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -605,6 +607,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -676,6 +679,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"known_ingredients_n" : 9,
Expand Down
7 changes: 7 additions & 0 deletions ...ed_test_results/api_v2_product_read/get-fields-attribute-groups-all-knowledge-panels.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -902,6 +902,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -920,6 +921,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -929,6 +931,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -1000,6 +1003,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"knowledge_panels" : {
Expand Down
7 changes: 7 additions & 0 deletions tests/integration/expected_test_results/api_v2_product_read/get-fields-raw.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -591,6 +592,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -600,6 +602,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -671,6 +674,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"known_ingredients_n" : 9,
Expand Down
7 changes: 7 additions & 0 deletions ...ntegration/expected_test_results/api_v2_product_write/get-product-auth-good-password.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,13 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:salt",
"en:sugar",
"en:water",
"en:wheat-flour"
],
"ingredients_without_ciqual_codes_n" : 4,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"known_ingredients_n" : 10,
Expand Down
7 changes: 7 additions & 0 deletions tests/integration/expected_test_results/api_v2_product_write/get-product.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,13 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:salt",
"en:sugar",
"en:water",
"en:wheat-flour"
],
"ingredients_without_ciqual_codes_n" : 4,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"known_ingredients_n" : 10,
Expand Down
7 changes: 7 additions & 0 deletions tests/integration/expected_test_results/api_v3_product_read/get-existing-product.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,6 +579,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -597,6 +598,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -606,6 +608,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -677,6 +680,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"known_ingredients_n" : 9,
Expand Down
7 changes: 7 additions & 0 deletions ...ntegration/expected_test_results/api_v3_product_read/get-fields-all-knowledge-panels.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,6 +579,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -597,6 +598,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -606,6 +608,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -677,6 +680,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"knowledge_panels" : {
Expand Down
7 changes: 7 additions & 0 deletions tests/integration/expected_test_results/api_v3_product_read/get-fields-all.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,6 +579,7 @@
],
"ingredients" : [
{
"ciqual_food_code" : "13050",
"id" : "en:apple",
"percent_estimate" : 62.5,
"percent_max" : 100,
Expand All @@ -597,6 +598,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "22000",
"id" : "en:egg",
"percent_estimate" : 9.375,
"percent_max" : 33.3333333333333,
Expand All @@ -606,6 +608,7 @@
"vegetarian" : "yes"
},
{
"ciqual_food_code" : "16129",
"from_palm_oil" : "yes",
"id" : "en:palm-oil",
"percent_estimate" : 9.375,
Expand Down Expand Up @@ -677,6 +680,10 @@
"ingredients_with_specified_percent_sum" : 0,
"ingredients_with_unspecified_percent_n" : 4,
"ingredients_with_unspecified_percent_sum" : 100,
"ingredients_without_ciqual_codes" : [
"en:milk"
],
"ingredients_without_ciqual_codes_n" : 1,
"interface_version_created" : "20150316.jqm2",
"interface_version_modified" : "20150316.jqm2",
"known_ingredients_n" : 9,
Expand Down
Loading

0 comments on commit c1ebb27

Please sign in to comment.