feat: Fix and improve detection of apps (name and UUID) to populate d…

…ata sources (openfoodfacts#6319)

* improve app and uuid detection in changes

* better tests

* better identification of apps

* add support for passing app_name and app_uuid through the API

* remove unused module

* add Data::DeepAccess

lib/ProductOpener/Config_off.pm

@@ -902,36 +902,62 @@ $options{display_tag_additives} = [
 
 ];
 
+# Used in the data_sources field (e.g. "App - Open Food Facts")
+$options{apps_names} = {
+
+	"elcoco" => "El CoCo",
+	"ethic-advisor" => "Ethic-Advisor",
+	"horizon" => "Horizon",
+	"infood" => "InFood",
+	"isve" => "IsVe",
+	"labeleat" => "LabelEat",
+	"off" => "Open Food Facts",
+	"scanfood" => "Scanfood",
+	"speisekammer" => "Speisekammer",
+	"waistline" => "Waistline",
+	"yuka" => "Yuka"
+};
 
 # Specific users used by apps
 $options{apps_userids} = {
 
-	"ethic-advisor" => "ethic-advisor",
+	"averment" => "isve",
 	"elcoco" => "elcoco",
+	"ethic-advisor" => "ethic-advisor",
+	"inf" => "infood",
 	"kiliweb" => "yuka",
 	"labeleat" => "labeleat",
+	"prepperapp" => "speisekammer",
+	"scanfood" => "scanfood",
+	"swipe-studio" => "horizon",
 	"waistline-app" => "waistline",
-	"inf" => "infood",
 };
 
+$options{official_app_id} = "off";
+$options{official_app_comment} = "(official (off|open food facts|openfoodfacts)|(off|open food facts|openfoodfacts) (official )?(android |ios )?(official )?app)";
+
 # (app)Official Android app 3.1.5 ( Added by 58abc55ceb98da6625cee5fb5feaf81 )
 # (app)Labeleat1.0-SgP5kUuoerWvNH3KLZr75n6RFGA0
 # (app)Contributed using: OFF app for iOS - v3.0 - user id: 3C0154A0-D19B-49EA-946F-CC33A05E404A
-# (app)Official Android app 3.1.5 ( Added by 58abc55ceb98da6625cee5fb5feaf81 )
 # (app)EthicAdvisorApp-production-2.6.3-user_17cf91e3-52ee-4431-aebf-7d455dd610f0
 # (app)El Coco - user b0e8d6a858034cc750136b8f19a8953d
 
+# app_uuid_prefix must be present to recognize the uuid, if the comment starts with the uuid, put an empty string
 $options{apps_uuid_prefix} = {
 
 	"elcoco" => " user ",
 	"ethic-advisor" => "user_",
-	"kiliweb" => "User :",
 	"labeleat" => "Labeleat([^-]*)-",
-	"waistline-app" => "Waistline:",
+	"off" => "added by",	
+	"scanfood" => "",
+	"waistline" => "Waistline:",
+	"yuka" => "User :",
 };
 
-$options{official_app_id} = "off";
-$options{official_app_comment} = "(official android app|off app)";
+$options{apps_uuid_suffix} = {
+
+	"scanfood" => "scanfood",
+};
 
 
 $options{nova_groups_tags} = {

lib/ProductOpener/Display.pm

@@ -183,6 +183,7 @@ use Excel::Writer::XLSX;
 use Template;
 use Data::Dumper;
 use Devel::Size qw(size total_size);
+use Data::DeepAccess qw(deep_get);
 use Log::Log4perl;
 
 use Log::Any '$log', default_adapter => 'Stderr';
@@ -3880,6 +3881,19 @@ HTML
 				display_error(lang("error_unknown_org"), 404);
 			}
 		}
+		elsif ($tagid =~ /\./) {
+			# App user (format "[app id].[app uuid]")
+
+			my $appid = $`;
+			my $uuid = $';
+
+			my $app_name = deep_get(\%options, "apps_names", $appid) || $appid;
+			my $app_user = f_lang("f_app_user", { app_name => $app_name });
+
+			$title = $app_user;
+			$products_title = $app_user;
+			$display_tag = $app_user;
+		}
 		else {
 
 			# User

lib/ProductOpener/Products.pm

@@ -134,6 +134,7 @@ use ProductOpener::Text qw/:all/;
 use CGI qw/:cgi :form escapeHTML/;
 use Encode;
 use Log::Any qw($log);
+use Data::DeepAccess qw(deep_get);
 
 use LWP::UserAgent;
 use Storable qw(dclone);
@@ -1088,14 +1089,34 @@ sub store_product($$$) {
 		delete $product_ref->{owners_tags};
 	}
 
-	push @{$changes_ref}, {
+	my $change_ref = {
 		userid => $user_id,
 		ip => remote_addr(),
 		t => $product_ref->{last_modified_t},
 		comment => $comment,
 		rev => $rev,
 	};
 
+	# Allow apps to send the user agent as a form parameter instead of a HTTP header, as some web based apps can't change the User-Agent header sent by the browser
+	my $user_agent = remove_tags_and_quote(decode utf8=>param("User-Agent"))
+		|| remove_tags_and_quote(decode utf8=>param("user-agent"))
+		|| remove_tags_and_quote(decode utf8=>param("user_agent"))
+		|| user_agent();
+
+	if ((defined $user_agent) and ($user_agent ne "")) {
+		$change_ref->{user_agent} = $user_agent;
+	}
+
+	# Allow apps to send app_name, app_version and app_uuid parameters
+	foreach my $field (qw(app_name app_version app_uuid)) {
+		my $value = remove_tags_and_quote(decode utf8=>param($field));
+		if ((defined $value) and ($value ne "")) {
+			$change_ref->{$field} = $value;
+		}
+	}
+
+	push @{$changes_ref}, $change_ref; 
+
 	add_user_teams($product_ref);
 
 	compute_codes($product_ref);
@@ -1106,7 +1127,7 @@ sub store_product($$$) {
 
 	compute_product_history_and_completeness($new_data_root, $product_ref, $changes_ref, $blame_ref);
 
-	compute_data_sources($product_ref);
+	compute_data_sources($product_ref, $changes_ref);
 
 	compute_main_countries($product_ref);
 
@@ -1132,7 +1153,7 @@ sub store_product($$$) {
 	# make sure nutrient values are numbers
 	make_sure_numbers_are_stored_as_numbers($product_ref);
 
-	my $change_ref = $changes_ref->[-1];
+	$change_ref = $changes_ref->[-1];
 	my $diffs = $change_ref->{diffs};
 	my %diffs = %{$diffs};
 	if ((!$diffs) or (!keys %diffs)) {
@@ -1167,13 +1188,21 @@ sub store_product($$$) {
 	return 1;
 }
 
-# Update the data-sources tag from the sources field
-# This function is for historic products, new sources should set the data_sources_tags field directly
-# through import_csv_file.pl / upload_photos.pl etc.
 
-sub compute_data_sources($) {
+=head2 compute_data_sources ( $product_ref, $changes_ref )
+
+Analyze the sources field of the product, as well as the changes to add to the data_sources field.
+
+Sources allows to add some producers imports that were done before the producers platform was created.
+
+The changes structure allows to add apps.
+
+=cut
+
+sub compute_data_sources($$) {
 
 	my $product_ref = shift;
+	my $changes_ref = shift;
 
 	my %data_sources = ();
 
@@ -1241,16 +1270,14 @@ sub compute_data_sources($) {
 
 	# Add a data source for apps
 
-	if (defined $product_ref->{editors_tags}) {
-		foreach my $editor (@{$product_ref->{editors_tags}}) {
-
-			if ($editor =~ /\./) {
+	foreach my $change_ref (@$changes_ref) {
+
+		if (defined $change_ref->{app}) {
 
-				my $app = $`;
+			my $app_name = deep_get(\%options, "apps_names", $change_ref->{app}) || $change_ref->{app};
 
-				$data_sources{"Apps"} = 1;
-				$data_sources{"App - $app"} = 1;
-			}
+			$data_sources{"Apps"} = 1;
+			$data_sources{"App - " . $app_name} = 1;
 		}
 	}
 
@@ -1476,22 +1503,45 @@ sub compute_completeness_and_missing_tags($$$) {
 }
 
 
+=head2 get_change_userid_or_uuid ( $change_ref )
+
+For a specific change, analyze change identifiers (comment, user agent, userid etc.)
+to determine if the change was done through an app, the OFF userid, or an app specific UUID
+
+=cut
+
 sub get_change_userid_or_uuid($) {
 
 	my $change_ref = shift;
 
 	my $userid = $change_ref->{userid};
 
-	my $app = "";
+	my $app;
+	my $app_userid_prefix;
 	my $uuid;
 
-	if ((defined $userid) and (defined $options{apps_userids}) and (defined $options{apps_userids}{$userid})) {
-		$app = $options{apps_userids}{$userid} . "\.";
+	# Is it an app that sent a app_name?
+	if (defined $change_ref->{app_name}) {
+		$app = get_string_id_for_lang("no_language", $change_ref->{app_name});
 	}
+	# or is the userid specific to an app?
+	elsif (defined $userid) {
+		$app = deep_get(\%options, "apps_userids", $userid);
+	}
+
+	# If the userid is an an account for an app, unset the userid,
+	# so that it can be replaced by the app + an app uuid if provided
+	if (defined $app) {
+		$userid = undef;
+	}
+	# Set the app field for the Open Food Facts app
 	elsif ((defined $options{official_app_comment}) and ($change_ref->{comment} =~ /$options{official_app_comment}/i)) {
-		$app = $options{official_app_id} . "\.";
+		$app = $options{official_app_id};
 	}
 
+	# If we do not have a user specific userid (e.g. a logged in user using the Open Food Facts app),
+	# try to identify the UUID passed in the comment by some apps
+
 	# use UUID provided by some apps like Yuka
 	# UUIDs are mix of [a-zA-Z0-9] chars, they must not be lowercased by getfile_id
 
@@ -1501,21 +1551,54 @@ sub get_change_userid_or_uuid($) {
 	#
 	# but not:
 	# (app)Updated via Power User Script
-	if ((defined $userid) and (defined $options{apps_uuid_prefix}) and (defined $options{apps_uuid_prefix}{$userid})
-		and ($change_ref->{comment} =~ /$options{apps_uuid_prefix}{$userid}/i)) {
-		$uuid = $';
-	}
 
-	if ((defined $uuid) and ($uuid !~ /^(\s|-|_|\.)*$/)) {
-		$uuid =~ s/^(\s*)//;
-		$uuid =~ s/(\s*)$//;
-		$userid = $app . $uuid;
+	if ((defined $app) and ((not defined $userid) or ($userid eq ''))) {
+
+		$app_userid_prefix = deep_get(\%options, "apps_uuid_prefix", $app);
+
+		# Check if the app passed the app_uuid parameter
+		if (defined $change_ref->{app_uuid}) {
+			$uuid = $change_ref->{app_uuid};
+		}
+		# Extract UUID from comment
+		elsif ((defined $app_userid_prefix)
+			and ($change_ref->{comment} =~ /$app_userid_prefix/i)) {
+			$uuid = $';
+		}
+
+		if (defined $uuid) {
+
+			# Remove any app specific suffix
+			my $app_userid_suffix = deep_get(\%options, "apps_uuid_suffix", $app);
+			if (defined $app_userid_suffix) {
+				$uuid =~ s/$app_userid_suffix(\s|\(|\[])*$//i;
+			}
+
+			$uuid =~ s/^(-|_|\s|\(|\[])+//;
+			$uuid =~ s/(-|_|\s|\)|\])+$//;
+		}
+
+		# If we have a uuid from an app, make the userid a combination of app + uuid
+		if ((defined $uuid) and ($uuid !~ /^(-|_|\s|-|_|\.)*$/)) {
+			$userid = $app . '.' . $uuid;
+		}
+		# otherwise use the original userid used for the API if any
+		elsif (defined $change_ref->{userid}) {
+			$userid = $change_ref->{userid};
+		}	
 	}
 
-	if ((not defined $userid) or ($userid eq '')) {
+	if (not defined $userid) {
 		$userid = "openfoodfacts-contributors";
+	}	
+
+	# Add the app to the change structure if we identified one, this will be used to populate the data sources field
+	if (defined $app) {
+		$change_ref->{app} = $app;
 	}
 
+	$log->debug("get_change_userid_or_uuid", { change_ref => $change_ref, app => $app, app_userid_prefix => $app_userid_prefix, uuid => $uuid, userid => $userid } ) if $log->is_debug();
+
 	return $userid;
 }
 

lib/ProductOpener/Tags.pm

@@ -1756,8 +1756,11 @@ sub generate_tags_taxonomy_extract ($$$$) {
 	my $options_ref = shift;
 	my $lcs_ref = shift;
 
+	$log->debug("generate_tags_taxonomy_extract", {tagtype => $tagtype, tags_ref => $tags_ref, options_ref => $options_ref, lcs_ref => $lcs_ref }) if $log->is_debug();
+
 	# Return empty hash if the taxonomy does not exist
 	if (not defined $translations_to{$tagtype}) {
+		$log->debug("taxonomy not found", {tagtype => $tagtype}) if $log->is_debug();
 		return {};
 	}
 

po/common/common.pot

@@ -5960,3 +5960,8 @@ msgstr "This might not be the answer people want to hear, but there is no safe l
 msgctxt "source"
 msgid "Source"
 msgstr "Source"
+
+# variable names between { } must not be translated
+msgctxt "f_app_user"
+msgid "A user of the {app_name} app"
+msgstr "A user of the {app_name} app"

po/common/en.po

@@ -5988,3 +5988,8 @@ msgstr "This might not be the answer people want to hear, but there is no safe l
 msgctxt "source"
 msgid "Source"
 msgstr "Source"
+
+# variable names between { } must not be translated
+msgctxt "f_app_user"
+msgid "A user of the {app_name} app"
+msgstr "A user of the {app_name} app"

scripts/update_all_products.pl

@@ -526,7 +526,7 @@
 					$product_ref->{rev} = $last_rev;
 					my $blame_ref = {};
 					compute_product_history_and_completeness($data_root, $product_ref, $changes_ref, $blame_ref);
-					compute_data_sources($product_ref);
+					compute_data_sources($product_ref, $changes_ref);
 					store("$data_root/products/$path/changes.sto", $changes_ref);
 				}
 				else {
@@ -831,7 +831,11 @@
 		}
 
 		if ($compute_data_sources) {
-			compute_data_sources($product_ref);
+			my $changes_ref = retrieve("$data_root/products/$path/changes.sto");
+			if (not defined $changes_ref) {
+				$changes_ref = [];
+			}
+			compute_data_sources($product_ref, $changes_ref);
 		}
 
 		if ($compute_nova) {
@@ -1016,7 +1020,7 @@
 			}
 			my $blame_ref =  {};
 			compute_product_history_and_completeness($data_root, $product_ref, $changes_ref, $blame_ref);
-			compute_data_sources($product_ref);
+			compute_data_sources($product_ref, $changes_ref);
 			store("$data_root/products/$path/changes.sto", $changes_ref);
 		}