fix: fix integration tests

lib/ProductOpener/APITest.pm

@@ -515,7 +515,18 @@ sub execute_api_tests ($file, $tests_ref, $ua = undef) {
 
 		my $response_content = $response->decoded_content;
 
-		if (not((defined $test_ref->{expected_type}) and ($test_ref->{expected_type} eq "html"))) {
+		if ($test_ref->{expected_type} eq 'text') {
+			# Check that the text file is the same as expected (useful for checking dynamic robots.txt)
+			is(
+				compare_file_to_expected_results(
+					$response_content, "$expected_result_dir/$test_case.txt",
+					$update_expected_results, $test_ref
+				),
+				1,
+				"$test_case - result"
+			);
+		}
+		elsif (not((defined $test_ref->{expected_type}) and ($test_ref->{expected_type} eq "html"))) {
 
 			# Check that we got a JSON response
 

lib/ProductOpener/Display.pm

@@ -282,7 +282,7 @@ if (defined $options{export_limit}) {
 }
 
 # Save all tag types to index in a set to make checks easier
-%index_tag_types_set;
+%index_tag_types_set = ();
 @index_tag_types_set{@ProductOpener::Config::index_tag_types} = ();
 
 # Initialize the Template module

lib/ProductOpener/Test.pm

@@ -37,6 +37,7 @@ BEGIN {
 		&capture_ouputs
 		&compare_arr
 		&ensure_expected_results_dir
+		&compare_file_to_expected_results
 		&compare_to_expected_results
 		&compare_array_to_expected_results
 		&compare_csv_file_to_expected_results
@@ -381,6 +382,65 @@ sub compare_to_expected_results ($object_ref, $expected_results_file, $update_ex
 	return 1;
 }
 
+=head2 compare_file_to_expected_results($content_str, $expected_results_file, $update_expected_results, $test_ref = undef) {
+
+Compare an string (e.g. text or HTML file) to expected results.
+
+The expected result is stored as a plain text file.
+
+This is so that we can easily see diffs with git diffs.
+
+=head3 Arguments
+
+=head4 $content_str - the reference string
+
+=head4 $expected_results_file - path to the file with stored results
+
+=head4 $update_expected_results - flag to indicate to save test results as expected results
+
+Tests will always pass when this flag is passed,
+and the new expected results can be diffed / committed in GitHub.
+
+=head4 $test_ref - an optional reference to an object describing the test case
+
+If the test fail, the test reference will be output in the C<diag>
+
+=cut
+
+sub compare_file_to_expected_results ($content_str, $expected_results_file, $update_expected_results, $test_ref = undef)
+{
+	my $desc = undef;
+	if (defined $test_ref) {
+		$desc = $test_ref->{desc} // $test_ref->{id};
+	}
+
+	if ($update_expected_results) {
+		open(my $result, ">:encoding(UTF-8)", $expected_results_file)
+			or confess("Could not create $expected_results_file: $!");
+		print $result $content_str;
+		close($result);
+	}
+	else {
+		# Compare the result with the expected result
+
+		if (open(my $IN, "<:encoding(UTF-8)", $expected_results_file)) {
+			my $expected_result = join('', (<$IN>));
+			my $title;
+			if ($test_ref && (ref($test_ref) eq "HASH")) {
+				$title = $test_ref->{desc} // $test_ref->{test_case} // $test_ref->{id};
+				$title = undef unless $title;
+			}
+			is($content_str, $expected_result, $title);
+		}
+		else {
+			fail("could not load $expected_results_file");
+			diag(explain $test_ref, explain $content_str);
+		}
+	}
+
+	return 1;
+}
+
 =head2 compare_csv_file_to_expected_results($csv_file, $expected_results_dir, $update_expected_results)
 
 Compare a CSV file containing product data (e.g. the result of a CSV export) to expected results.

tests/integration/expected_test_results/page_crawler/get-robots-txt-ch-it.txt

@@ -0,0 +1,211 @@
+User-agent: *
+Allow: /cgi/product_image.pl
+Allow: /cgi/opensearch.pl
+Disallow: /cgi
+Disallow: /code
+Disallow: /api
+Disallow: /additives
+Disallow: /additives-debug
+Disallow: /additives-next
+Disallow: /additives-prev
+Disallow: /allergen
+Disallow: /allergens
+Disallow: /amino-acid
+Disallow: /amino-acids
+Disallow: /brands
+Disallow: /categories
+Disallow: /categories-debug
+Disallow: /categories-next
+Disallow: /categories-prev
+Disallow: /categories-properties
+Disallow: /checker
+Disallow: /checkers
+Disallow: /city
+Disallow: /cities
+Disallow: /code
+Disallow: /codes
+Disallow: /corrector
+Disallow: /correctors
+Disallow: /country
+Disallow: /countries
+Disallow: /data-quality
+Disallow: /data-quality-bug
+Disallow: /data-quality-bugs
+Disallow: /data-quality-error
+Disallow: /data-quality-errors
+Disallow: /data-quality-error-producers
+Disallow: /data-quality-errors-producers
+Disallow: /data-quality-info
+Disallow: /data-quality-warning
+Disallow: /data-quality-warnings
+Disallow: /data-quality-warning-producers
+Disallow: /data-quality-warnings-producers
+Disallow: /data-source
+Disallow: /data-sources
+Disallow: /debug
+Disallow: /editor
+Disallow: /editors
+Disallow: /packager-code
+Disallow: /packager-codes
+Disallow: /entry-date
+Disallow: /entry-dates
+Disallow: /food-group
+Disallow: /food-groups
+Disallow: /import
+Disallow: /imports
+Disallow: /possible-improvement
+Disallow: /possible-improvements
+Disallow: /informer
+Disallow: /informers
+Disallow: /ingredient
+Disallow: /ingredients
+Disallow: /ingredients-analysis
+Disallow: /ingredients-from-palm-oil
+Disallow: /number-of-ingredients
+Disallow: /numbers-of-ingredients
+Disallow: /ingredients-that-may-be-from-palm-oil
+Disallow: /known-nutrient
+Disallow: /known-nutrients
+Disallow: /labels
+Disallow: /labels-debug
+Disallow: /labels-next
+Disallow: /labels-prev
+Disallow: /language
+Disallow: /languages
+Disallow: /last-check-date
+Disallow: /last-check-dates
+Disallow: /last-edit-date
+Disallow: /last-edit-dates
+Disallow: /last-image-date
+Disallow: /last-image-dates
+Disallow: /manufacturing-place
+Disallow: /manufacturing-places
+Disallow: /mineral
+Disallow: /minerals
+Disallow: /misc
+Disallow: /mission
+Disallow: /missions
+Disallow: /nova-groups
+Disallow: /nucleotide
+Disallow: /nucleotides
+Disallow: /nutrient-level
+Disallow: /nutrient-levels
+Disallow: /nutrient
+Disallow: /nutrients
+Disallow: /nutrition-grades
+Disallow: /origin
+Disallow: /origins
+Disallow: /other-nutritional-substance
+Disallow: /other-nutritional-substances
+Disallow: /owner
+Disallow: /owners
+Disallow: /packaging
+Disallow: /packaging-materials
+Disallow: /packaging-recycling
+Disallow: /packaging-shape
+Disallow: /packaging-shapes
+Disallow: /period-after-opening
+Disallow: /periods-after-opening
+Disallow: /photographer
+Disallow: /photographers
+Disallow: /pnns-group-1
+Disallow: /pnns-groups-1
+Disallow: /pnns-group-2
+Disallow: /pnns-groups-2
+Disallow: /popularity
+Disallow: /purchase-place
+Disallow: /purchase-places
+Disallow: /quality
+Disallow: /state
+Disallow: /states
+Disallow: /store
+Disallow: /stores
+Disallow: /team
+Disallow: /teams
+Disallow: /trace
+Disallow: /traces
+Disallow: /unknown-nutrient
+Disallow: /unknown-nutrients
+Disallow: /contributor
+Disallow: /contributors
+Disallow: /vitamin
+Disallow: /vitamins
+Disallow: /weigher
+Disallow: /weighers
+
+
+# Disallow: Bingbot (temporary test)
+User-agent: Bingbot
+Disallow: /
+
+# Disallow: SEOkicks-Robot
+User-agent: SEOkicks-Robot
+Disallow: /
+
+# http://www.opensiteexplorer.org/dotbot
+User-agent: DotBot
+Disallow: /
+User-agent: dotbot
+Disallow: /
+
+# http://www.searchmetrics.com
+User-agent: SearchmetricsBot
+Disallow: /
+
+# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
+User-agent: MJ12bot
+Disallow: /
+
+# http://www.domaintools.com/webmasters/surveybot.php
+User-agent: SurveyBot
+Disallow: /
+
+# http://www.seodiver.com/bot
+User-agent: SEOdiver
+Disallow: /
+
+# http://openlinkprofiler.org/bot
+User-agent: spbot
+Disallow: /
+
+# http://www.wotbox.com/bot/
+User-agent: wotbox
+Disallow: /
+
+User-agent: Cliqzbot/3.0
+Disallow: /
+User-agent: Cliqzbot
+Disallow: /
+User-agent: SeekportBot
+Disallow: /
+User-agent: Seekport Bot
+Disallow: /
+User-agent: Seekport
+Disallow: /
+User-agent: Paracrawl
+Disallow: /
+User-agent: Scrapy/1.5.0
+Disallow: /
+User-agent: Scrapy
+Disallow: /
+User-agent: VelenPublicWebCrawler (velen.io)
+Disallow: /
+User-agent: VelenPublicWebCrawler
+Disallow: /
+User-agent: SemrushBot/2~bl
+Disallow: /
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: MegaIndex.ru/2.0
+Disallow: /
+User-agent: MegaIndex.ru
+Disallow: /
+User-agent: YandexMarket
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /