test: integration test on import_csv (#6245)

.gitignore

@@ -5,6 +5,9 @@ lib/ProductOpener/SiteLang.pm
 taxonomies/nutrient_levels.txt
 po/site-specific
 
+# tests outputs
+t/outputs/
+
 # Local files
 .DS_Store?
 ._*

cpanfile

@@ -35,6 +35,7 @@ requires 'Excel::Writer::XLSX', '>= 1.09'; # libexcel-writer-xlsx-perl
 requires 'Pod::Simple::HTMLBatch'; # libpod-simple-perl
 requires 'GeoIP2', '>= 2.006002, < 3.0'; # libgeoip2-perl, deps: libdata-validate-ip-perl libio-compress-perl libjson-maybexs-perl liblist-someutils-perl, libdata-dumper-concise-perl, libdata-printer-perl
 requires 'Email::Valid', '>= 1.202, < 2.0'; # libemail-valid-perl
+requires 'Path::Tiny', '>= 0.118'; # libpath-tiny-perl
 
 # Probably not available as Debian/Ubuntu packages
 requires 'MongoDB', '>= 2.2.2, < 2.3'; # libmongodb-perl has 1.8.1/2.0.3 vs 2.2.2. deps: libauthen-sasl-saslprep-perl, libbson-perl, libauthen-scram-perl, libclass-xsaccessor-perl, libdigest-hmac-perl, libsafe-isa-perl, libconfig-autoconf-perl, libpath-tiny-perl
@@ -81,8 +82,12 @@ requires 'Action::Retry'; # deps: libmath-fibonacci-perl
 
 on 'test' => sub {
   requires 'Test::More', '>= 1.302186, < 2.0';
+  requires 'Test::MockModule';
+  requires 'Mock::Quick';
   requires 'Test::Number::Delta'; # libtest-number-delta-perl
   requires 'Log::Any::Adapter::TAP'; # liblog-any-adapter-tap-perl
+  requires 'IO::Capture::Stdout::Extended';
+  requires 'IO::Capture::Stderr::Extended';
 };
 
 on 'develop' => sub {

lib/ProductOpener/Import.pm

@@ -230,6 +230,20 @@ sub import_images_from_dir($$) {
 	return $images_ref;
 }
 
+# download image at given url parameter
+sub download_image($) {
+	my $image_url = shift;
+
+	require LWP::UserAgent;
+
+	my $ua = LWP::UserAgent->new(timeout => 10);
+
+	# Some platforms such as CloudFlare block the default LWP user agent.
+	$ua->agent(lang('site_name') . " (https://$server_domain)");
+
+	return $ua->get($image_url);
+}
+
 
 # deduplicate column names
 # We may have duplicate columns (e.g. image_other_url),
@@ -470,7 +484,7 @@ sub import_csv_file($) {
 
 	# first line contains headers
 	my $columns_ref = $csv->getline ($io);
-	$csv->column_names (@{deduped_colnames($columns_ref)});
+	$csv->column_names(@{deduped_colnames($columns_ref)});
 
 	my $i = 0;
 	my $j = 0;
@@ -1929,14 +1943,7 @@ sub import_csv_file($) {
 
 									$log->debug("download image file", { file => $file, image_url => $image_url }) if $log->is_debug();
 
-									require LWP::UserAgent;
-
-									my $ua = LWP::UserAgent->new(timeout => 10);
-
-									# Some platforms such as CloudFlare block the default LWP user agent.
-									$ua->agent(lang('site_name') . " (https://$server_domain)");
-
-									my $response = $ua->get($image_url);
+									my $response = download_image($image_url);
 
 									if ($response->is_success) {
 										$log->debug("downloaded image file", { file => $file }) if $log->is_debug();

lib/ProductOpener/Producers.pm

@@ -653,6 +653,18 @@ my %in = (
 	"fr" => "en",
 );
 
+=head2 init_fields_columns_names_for_lang ( $l )
+
+Populates global $fields_columns_names_for_lang
+for the specified language.
+
+=head3 Arguments
+
+=head4 $l - required
+
+Language code (string)
+
+=cut
 
 sub init_fields_columns_names_for_lang($) {
 
@@ -1148,6 +1160,7 @@ sub init_columns_fields_match($$) {
 	init_fields_columns_names_for_lang($lc);
 
 	if ($lc ne "en") {
+		# we also add english names
 		init_fields_columns_names_for_lang("en");
 	}