support multiple languages in import files on producers platform (#5990)

* support multiple languages in import files on producers platform #5989 * Update cgi/import_file_select_format.pl Co-authored-by: Alex Garel <[email protected]> * refactor generation of the list of languages select options * remove unused code, generate the regexp for language specific fields * small fix + comments from Alex * suggestions from Alex * use the right template data ref Co-authored-by: Alex Garel <[email protected]>
openfoodfacts · Oct 21, 2021 · 40eb397 · 40eb397
1 parent 5278808
commit 40eb397
Show file tree

Hide file tree

Showing 7 changed files with 165 additions and 87 deletions.
diff --git a/cgi/import_file_select_format.pl b/cgi/import_file_select_format.pl
@@ -36,6 +36,8 @@
 use ProductOpener::Lang qw/:all/;
 use ProductOpener::Mail qw/:all/;
 use ProductOpener::Producers qw/:all/;
+use ProductOpener::Tags qw(%language_fields display_taxonomy_tag);
+use ProductOpener::Web qw(get_languages_options_list);
 
 use Apache2::RequestRec ();
 use Apache2::Const ();
@@ -174,6 +176,10 @@
 	$template_data_ref->{field_on_site} = $field_on_site;
 	$template_data_ref->{file_id} = $file_id;
 
+	# List of all languages for the template to display a dropdown for fields that are language specific
+	$template_data_ref->{lang_options} = get_languages_options_list($lc);
+	$template_data_ref->{language_fields} = [ keys %language_fields ];
+
 	process_template('web/pages/import_file_select_format/import_file_select_format.tt.html', $template_data_ref, \$html);
 	process_template('web/pages/import_file_select_format/import_file_select_format.tt.js', $template_data_ref, \$js);
 	$initjs .= $js;

diff --git a/cgi/product_multilingual.pl b/cgi/product_multilingual.pl
@@ -43,6 +43,7 @@
 use ProductOpener::Ecoscore qw/:all/;
 use ProductOpener::Packaging qw/:all/;
 use ProductOpener::ForestFootprint qw/:all/;
+use ProductOpener::Web qw(get_languages_options_list);
 
 use Apache2::RequestRec ();
 use Apache2::Const ();
@@ -937,26 +938,14 @@ ($$$)
 	}
 
 	# Main language
-	my @lang_options;
-	my @lang_values = sort { display_taxonomy_tag($lc,'languages',$language_codes{$a}) cmp display_taxonomy_tag($lc,'languages',$language_codes{$b})} @Langs;
-
-	my %lang_labels = ();
-	foreach my $l (@lang_values) {
-		next if (length($l) > 2);
-		$lang_labels{$l} = display_taxonomy_tag($lc,'languages',$language_codes{$l});
-		push(@lang_options, {
-			value => $l,
-			label => $lang_labels{$l},
-		});
-	}
-
 	my $lang_value = $lang;
 	if (defined $product_ref->{lc}) {
 		$lang_value = $product_ref->{lc};
 	}
 
 	$template_data_ref_display->{product_lang_value} = $lang_value;
-	$template_data_ref_display->{lang_options} = \@lang_options;
+	# List of all languages for the template to display a dropdown for fields that are language specific
+	$template_data_ref_display->{lang_options} = get_languages_options_list($lc);
 	$template_data_ref_display->{display_select_manage} = display_select_manage($product_ref);
 
 	# sort function to put main language first, other languages by alphabetical order, then add new language tab

diff --git a/html/js/product-multilingual.js b/html/js/product-multilingual.js
@@ -936,15 +936,9 @@ function initLanguageAdding() {
 
   const existingLanguages = [];
   const tabs = document.querySelectorAll('li.tabs:not([data-language="new_lc"]):not(.tabs_new)');
-  // eslint-disable-next-line guard-for-in
-  for (let i = 0; i < tabs.length; ++i) {
-    existingLanguages.push(tabs[i].dataset.language);
-  }
+  tabs.forEach((tab) => existingLanguages.push(tab.dataset.language));
 
-  // eslint-disable-next-line no-unused-vars
-  const unusedLanguages = languages.filter(function(value) {
-    return !existingLanguages.includes(value.id);
-  });
+  const unusedLanguages = languages.filter((value) => !existingLanguages.includes(value.id));
 
   $(".select_add_language").select2({
     placeholder: placeholder,

diff --git a/lib/ProductOpener/Producers.pm b/lib/ProductOpener/Producers.pm
@@ -419,6 +419,10 @@ sub convert_file($$$$) {
 					$field = undef;
 				}
 			}
+			# For language specific field, add the language code at the end of the field name
+			elsif ((defined $language_fields{$field}) and (defined $columns_fields_ref->{$column}{lc})) {
+				$field .= "_" . $columns_fields_ref->{$column}{lc};
+			}
 
 			$log->debug("convert_file - found matching column", { column => $column, field => $field, col => $col }) if $log->is_debug();
 
@@ -573,9 +577,9 @@ en => {
 	lc => ["lang"],
 	code => ["code", "codes", "barcodes", "barcode", "ean", "ean-13", "ean13", "gtin", "eans", "gtins", "upc", "ean/gtin1", "gencod", "gencods", "gencode", "gencodes", "ean-barcode","ean-barcode-number","ean-code"],
 	producer_product_id => ["internal code"],
-	product_name_en => ["name", "name of the product", "name of product", "product name", "product", "commercial name"],
+	product_name => ["name", "name of the product", "name of product", "product name", "product", "commercial name"],
 	carbohydrates_100g_value_unit => ["carbohydronate", "carbohydronates"], # yuka bug, does not exist
-	ingredients_text_en => ["ingredients", "ingredients list", "ingredient list", "list of ingredients"],
+	ingredients_text => ["ingredients", "ingredients list", "ingredient list", "list of ingredients"],
 	allergens => ["allergens", "allergens list", "allergen list", "list of allergens"],
 	traces => ["traces", "traces list", "trace list", "list of traces"],
 	nutriscore_grade_producer => ["nutri-score", "nutriscore"],
@@ -589,8 +593,8 @@ de => {
 
 es => {
 	code => ["Código de barras", "Códigos de barras"],
-	product_name_es => ["nombre", "nombre producto", "nombre del producto"],
-	ingredients_text_es => ["ingredientes", "lista ingredientes", "lista de ingredientes"],
+	product_name => ["nombre", "nombre producto", "nombre del producto"],
+	ingredients_text => ["ingredientes", "lista ingredientes", "lista de ingredientes"],
 	net_weight_value_unit => ["peso unitrario", "peso unitario"],   # Yuka
 	"energy-kcal_100g_value_unit" => ["calorias"],
 	"link" => ["Enlace a la página del producto en el sitio oficial del fabricante"],
@@ -601,10 +605,10 @@ fr => {
 	producer_product_id => ["code interne", "code int"],
 	categories => ["Catégorie(s)"],
 	brands => ["Marque(s)", "libellé marque"],
-	product_name_fr => ["nom", "nom produit", "nom du produit", "produit", "nom commercial", "dénomination", "dénomination commerciale", "dénomination marketing", "nom marketing", "libellé marketing", "libellé", "désignation"],
-	abbreviated_product_name_fr => ["nom abrégé", "nom abrégé du produit", "nom du produit abrégé", "nom du produit avec abbréviations"],
-	generic_name_fr => ["dénomination légale", "déno légale", "dénomination légale de vente"],
-	ingredients_text_fr => ["ingrédients", "ingredient", "liste des ingrédients", "liste d'ingrédients", "liste ingrédients", "listes d'ingrédients"],
+	product_name => ["nom", "nom produit", "nom du produit", "produit", "nom commercial", "dénomination", "dénomination commerciale", "dénomination marketing", "nom marketing", "libellé marketing", "libellé", "désignation"],
+	abbreviated_product_name => ["nom abrégé", "nom abrégé du produit", "nom du produit abrégé", "nom du produit avec abbréviations"],
+	generic_name => ["dénomination légale", "déno légale", "dénomination légale de vente"],
+	ingredients_text => ["ingrédients", "ingredient", "liste des ingrédients", "liste d'ingrédients", "liste ingrédients", "listes d'ingrédients"],
 	allergens => ["Substances ou produits provoquant des allergies ou intolérances", "Allergènes et Traces Potentielles", "allergènes et traces"],
 	traces => ["Traces éventuelles"],
 	image_front_url_fr => ["visuel", "photo", "photo produit"],
@@ -614,18 +618,18 @@ fr => {
 	volume_value_unit => ["volume net"],
 	net_weight_value_unit => ["poids"],
 	drained_weight_value_unit => ["poids net égoutté"],
-	recycling_instructions_to_recycle_fr => ["à recycler", "consigne à recycler"],
-	recycling_instructions_to_discard_fr => ["à jeter", "consigne à jeter"],
-	conservation_conditions_fr => ["Conditions de conservation et d'utilisation"],
-	preparation_fr => ["conseils de préparation", "instructions de préparation", "Mode d'emploi"],
+	recycling_instructions_to_recycle => ["à recycler", "consigne à recycler"],
+	recycling_instructions_to_discard => ["à jeter", "consigne à jeter"],
+	conservation_conditions => ["Conditions de conservation et d'utilisation"],
+	preparation => ["conseils de préparation", "instructions de préparation", "Mode d'emploi"],
 	link => ["lien", "lien du produit", "lien internet", "lien vers la page internet"],
 	manufacturing_places => ["lieu de conditionnement", "lieux de conditionnement", "lieu de fabrication", "lieux du fabrication", "lieu de fabrication du produit"],
 	nutriscore_grade_producer => ["note nutri-score", "note nutriscore", "lettre nutri-score", "lettre nutriscore"],
 	nutriscore_score_producer => ["score nutri-score", "score nutritionnel"],
 	emb_codes => ["estampilles sanitaires / localisation", "codes emballeurs / localisation"],
 	lc => ["langue", "langue du produit"],
 	obsolete => ["Le produit n'est plus en vente.", "Produit retiré de la vente", "Produit obsolète", "Obsolète"],
-	packaging_text_fr => ["Instruction de recyclage et/ou information d'emballage"],
+	packaging_text => ["Instruction de recyclage et/ou information d'emballage"],
 },
 
 );
@@ -972,17 +976,17 @@ sub init_other_fields_columns_names_for_lang($) {
 					foreach my $field_l ($l, "en") {
 
 						my @synonyms = ($field, $Lang{$field}{$field_l});
-						if ((defined $fields_synonyms{$field_l}) and (defined $fields_synonyms{$field_l}{$field . "_" . $field_l})) {
-							foreach my $synonym (@{$fields_synonyms{$field_l}{$field . "_" . $field_l}}) {
+						if ((defined $fields_synonyms{$field_l}) and (defined $fields_synonyms{$field_l}{$field})) {
+							foreach my $synonym (@{$fields_synonyms{$field_l}{$field}}) {
 								push @synonyms, $synonym;
 							}
 						}
 
 						foreach my $synonym (@synonyms) {
-							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym)} = {field => $field . "_$l"};
-							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym . " " . $l)} = {field => $field . "_$l"};
-							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym . " " . $language_codes{$l})} = {field => $field . "_$l"};
-							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym . " " . display_taxonomy_tag($l,'languages',$language_codes{$l}))} = {field => $field . "_$l"};
+							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym)} = {field => $field, lc => $l};
+							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym . " " . $l)} = {field => $field , lc => $l};
+							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym . " " . $language_codes{$l})} = {field => $field, lc => $l};
+							$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym . " " . display_taxonomy_tag($field_l,'languages',$language_codes{$l}))} = {field => $field, lc => $l};
 						}
 					}
 				}
@@ -1018,9 +1022,12 @@ sub init_other_fields_columns_names_for_lang($) {
 	# Extra synonyms
 	if (defined $fields_synonyms{$l}) {
 		foreach my $field (keys %{$fields_synonyms{$l}}) {
-			foreach my $synonym (@{$fields_synonyms{$l}{$field}}) {
-				# $log->debug("synonyms", { l=>$l, field=>$field, synonym=>$synonym }) if $log->is_debug();
-				$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym) } = {field => $field};
+			# Language specific fields synonyms have already been included
+			if (not defined $language_fields{$field}) {
+				foreach my $synonym (@{$fields_synonyms{$l}{$field}}) {
+					# $log->debug("synonyms", { l=>$l, field=>$field, synonym=>$synonym }) if $log->is_debug();
+					$fields_columns_names_for_lang{$l}{get_string_id_for_lang("no_language", $synonym) } = {field => $field};
+				}
 			}
 		}
 	}
@@ -1055,6 +1062,20 @@ sub match_column_name_to_field($$) {
 			$results_ref = $fields_columns_names_for_lang{en}{$column_id};
 		}
 	}
+	# try to see if there is a trailing language code
+	# e.g. ingredients-text-fr 
+	elsif ($column_id =~ /-([a-z]{2})$/) {
+		$column_id = $`;
+		my $field_lc = $1;
+		if (defined $fields_columns_names_for_lang{$l}{$column_id}) {
+			$results_ref = dclone($fields_columns_names_for_lang{$l}{$column_id});
+			$results_ref->{lc} = $field_lc;
+		}
+		elsif (defined $fields_columns_names_for_lang{en}{$column_id}) {
+			$results_ref = dclone($fields_columns_names_for_lang{en}{$column_id});
+			$results_ref->{lc} = $field_lc;
+		}
+	}
 
 	return $results_ref;
 }
@@ -1388,7 +1409,7 @@ JSON
 
 				$log->debug("Select2 option", { group_id => $group_id, field=>$field, name=>$name }) if $log->is_debug();
 
-				if ((defined $language_fields{$field}) or (($group_id eq "images") and ($field =~ /image_(front|ingredients|nutrition|packaging)/))) {
+				if (($group_id eq "images") and ($field =~ /image_(front|ingredients|nutrition|packaging)/)) {
 
 					foreach my $l (@{$lcs_ref}) {
 						my $language = "";    # Don't specify the language if there is just one

diff --git a/lib/ProductOpener/Web.pm b/lib/ProductOpener/Web.pm
@@ -65,6 +65,7 @@ BEGIN
 		&display_data_quality_issues_and_improvement_opportunities
 		&display_data_quality_description
 		&display_knowledge_panel
+		&get_languages_options_list
 		); #the fucntions which are called outside this file
 	%EXPORT_TAGS = (all => [@EXPORT_OK]);
 }
@@ -437,4 +438,36 @@ sub display_knowledge_panel($$) {
 	return $html;
 }
 
-1;
+
+=head2 get_languages_options_list( $target_lc )
+
+Generates a data structure containing the list of languages and their translation in a target language.
+The data structured can be passed to HTML templates to construction a list of options for a select element.
+
+=cut
+
+sub get_languages_options_list($) {
+
+	my $target_lc = shift;
+
+	my @lang_options = ();
+
+	my %lang_labels = ();
+	foreach my $l (@Langs) {
+		$lang_labels{$l} = display_taxonomy_tag($target_lc,'languages',$language_codes{$l});
+	}
+
+	my @lang_values = sort { $lang_labels{$a} cmp $lang_labels{$b} } @Langs;
+
+	foreach my $l (@lang_values) {
+
+		push(@lang_options, {
+			value => $l,
+			label => $lang_labels{$l},
+		});
+	}
+
+	return \@lang_options;
+}
+
+1;