From 9121e87a0c37d82ad2cef69e715eb61a57667f99 Mon Sep 17 00:00:00 2001 From: Markus Friedrich Date: Fri, 6 Oct 2023 11:19:08 +0200 Subject: [PATCH] !!![TASK] Clean and optimize frontend helper: PageIndexer Cleans and optimizes the frontend helper "PageIndexer" integration test. Failure test for non existing pages/sites is refactored and PageIndexer adapted, even if AfterCacheableContentIsGeneratedEvent is not dispatched e.g. as the page is not found, the response will always contain the pageIndexed flag. Additionally the methods of PageIndexer that were defined as public to simplify testing are now protected, the tests are adapted accordingly. If you have used this methods, you have to adapt your code. Affected methods: - setupConfiguration - index - indexPage Relates: #3735 --- .../IndexQueue/FrontendHelper/PageIndexer.php | 28 ++- Documentation/Releases/solr-release-12-0.rst | 8 + .../Fixtures/Search/can_search.csv | 56 +----- .../Fixtures/Search/phrase_search.csv | 94 ++++------ .../Fixtures/Search/phrase_search_bigram.csv | 94 ++++------ .../Fixtures/Search/phrase_search_trigram.csv | 93 ++++------ .../can_index_custom_pagetype_into_solr.csv | 60 +------ .../Fixtures/can_index_into_solr.csv | 51 +----- .../Fixtures/can_index_mounted_page.csv | 47 +---- .../can_index_multiple_mounted_page.csv | 93 ---------- ...n_index_page_with_relation_to_category.csv | 53 +----- .../can_index_page_with_relation_to_page.csv | 60 +------ ...index_with_additional_fields_into_solr.csv | 54 +----- ...an_overwrite_configuration_in_rootline.csv | 62 +------ .../does_not_die_if_page_not_available.csv | 57 +----- .../FrontendHelper/PageIndexerTest.php | 162 ++++++++++++------ Tests/Integration/SearchTest.php | 65 +++---- 17 files changed, 288 insertions(+), 849 deletions(-) diff --git a/Classes/IndexQueue/FrontendHelper/PageIndexer.php b/Classes/IndexQueue/FrontendHelper/PageIndexer.php index f1ca95187c..39604bb6b8 100644 --- a/Classes/IndexQueue/FrontendHelper/PageIndexer.php +++ b/Classes/IndexQueue/FrontendHelper/PageIndexer.php @@ -192,9 +192,9 @@ public function __invoke(AfterCacheableContentIsGeneratedEvent $event): void } /** - * @internal currently only public for tests + * Index item */ - public function index(Item $indexQueueItem, TypoScriptFrontendController $tsfe): void + protected function index(Item $indexQueueItem, TypoScriptFrontendController $tsfe): void { $this->solrConnection = $this->getSolrConnection($indexQueueItem, $tsfe->getLanguage(), $this->configuration->getLoggingExceptions()); @@ -291,7 +291,7 @@ protected function getPageDocument(TypoScriptFrontendController $tsfe, string $u * * @return bool TRUE after successfully indexing the page, FALSE on error */ - public function indexPage( + protected function indexPage( Document $pageDocument, Item $indexQueueItem, TypoScriptFrontendController $tsfe, @@ -366,11 +366,13 @@ protected function addDocumentsToSolrIndex(array $documents): bool } /** - * @internal only used for tests + * Initialize PageIndexer + * + * As the Solr configuration initialization might affect the request + * we cannot initialize the configuration directly on activation */ - public function setupConfiguration(): void + protected function setupConfiguration(): void { - // currently needed for tests, will be separated. $this->logger = new SolrLogManager(__CLASS__, GeneralUtility::makeInstance(DebugWriter::class)); $this->configuration = Util::getSolrConfiguration(); } @@ -385,7 +387,19 @@ protected function getEventDispatcher(): EventDispatcherInterface */ public function deactivate(PageIndexerResponse $response): void { + if ($this->activated) { + if (!isset($this->responseData['pageIndexed'])) { + $this->responseData['pageIndexed'] = false; + } + $response->addActionResult($this->action, $this->responseData); + + $this->setupConfiguration(); + if ($this->configuration->getLoggingExceptions()) { + $this->logger->error( + 'Unknown exception while trying to index page', + ); + } + } $this->activated = false; - $response->addActionResult($this->action, $this->responseData); } } diff --git a/Documentation/Releases/solr-release-12-0.rst b/Documentation/Releases/solr-release-12-0.rst index 85c32b3bcd..7da353a67c 100644 --- a/Documentation/Releases/solr-release-12-0.rst +++ b/Documentation/Releases/solr-release-12-0.rst @@ -158,6 +158,14 @@ If you've used this class or the SolrConnection directly, you have to adapt your Note: With dropping the Node implementation we also dropped the backwards compatibility that allows to define the Solr path segment "/solr" within "solr_path_read" or "solr_path_write". Be sure your configuration doesn't contain this path segment! +!!! Changed visibility of ApacheSolrForTypo3\Solr\IndexQueue\FrontendHelper\PageIndexer methods +----------------------------------------------------------------------------------------------- + +For testing purposes some methods of the PageIndexer were defined as public, these methods are now protected. The tests are adapted accordingly, so that there is no need to declare the methods as public. +If you have used one of this methods, you have to adapt your code. Affected methods: +- setupConfiguration +- index +- indexPage !!! Solr route enhancer disabled by default ------------------------------------------- diff --git a/Tests/Integration/Fixtures/Search/can_search.csv b/Tests/Integration/Fixtures/Search/can_search.csv index d1b443e451..a88f9adf44 100644 --- a/Tests/Integration/Fixtures/Search/can_search.csv +++ b/Tests/Integration/Fixtures/Search/can_search.csv @@ -1,50 +1,6 @@ -"pages",,,,,, -,"uid","is_siteroot","doktype","title","slug","hidden" -,1,1,1,"Hello Search Test","/",0 -"sys_template",,,,,, -,"uid","pid","root","clear","sorting","config", -,1,1,1,3,100," -page = PAGE -page.typeNum = 0 - -plugin.tx_solr { - - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - - } - } -}" +"pages", +,"uid","pid","is_siteroot","doktype","slug","title","subtitle","crdate","tstamp" +,2,1,1,1,"/hello_solr","Hello Search Test","the subtitle",1449151778,1449151778 +"tx_solr_indexqueue_item", +,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" +,2,1,"pages",2,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/Fixtures/Search/phrase_search.csv b/Tests/Integration/Fixtures/Search/phrase_search.csv index 71701b7ce5..870a47b47c 100644 --- a/Tests/Integration/Fixtures/Search/phrase_search.csv +++ b/Tests/Integration/Fixtures/Search/phrase_search.csv @@ -1,64 +1,34 @@ "pages",,,,,,, ,"uid","pid","is_siteroot","doktype","hidden","slug","title" -,1,0,1,1,0,"/","World Hello Search Test" -,2,1,0,1,0,"/improve","Hello you can improve the precision by using phrase configuration World" -,3,1,0,1,0,"/hello-phrase-search-world","Hello phrase search World" -,4,1,0,1,0,"/hello-test-wonderful-world","Hello Test wonderful World" -,5,1,0,1,0,"/hello-wonderful-test-world","Hello wonderful Test World" -,6,1,0,1,0,"/bigram-phrases","Bigraming phrases strip down the sentence to the two-word combinations." -,7,1,0,1,0,"/trigram-phrases","Trigraming phrases strip down the sentence to triplets phrases" -,8,1,0,1,0,"/difference-to-bi-is-tri","Difference to bi is tri, which means building of triplets phrases." -,9,1,0,1,0,"/hello-solr-wolrd","Hello Solr Wolrd" -,10,1,0,1,0,"/hello-test-wolrd","Hello Test Wolrd" -,11,1,0,1,0,"/test-hello-solr-world","Test Hello Solr World" -,12,1,0,1,0,"/test-hello-search-world","Test Hello Search World" -,13,1,0,1,0,"/hello-the-test-search-world","Hello the Test Search World" -,14,1,0,1,0,"/hello-world-for-phrase-searching","Hello World for phrase searching" -,15,1,0,1,0,"/about","Solr is blazing-fast and open source enterprise search platform built on Apache Lucene" -"sys_template",,,,,,, -,"uid","pid","root","clear","sorting","constants","config", -,1,1,1,3,100,""," -page = PAGE -page.typeNum = 0 - -plugin.tx_solr { - search.query.phrase = 1 - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - - } - } -}", +,2,1,1,1,0,"/","World Hello Search Test" +,3,1,0,1,0,"/improve","Hello you can improve the precision by using phrase configuration World" +,4,1,0,1,0,"/hello-phrase-search-world","Hello phrase search World" +,5,1,0,1,0,"/hello-test-wonderful-world","Hello Test wonderful World" +,6,1,0,1,0,"/hello-wonderful-test-world","Hello wonderful Test World" +,7,1,0,1,0,"/bigram-phrases","Bigraming phrases strip down the sentence to the two-word combinations." +,8,1,0,1,0,"/trigram-phrases","Trigraming phrases strip down the sentence to triplets phrases" +,9,1,0,1,0,"/difference-to-bi-is-tri","Difference to bi is tri, which means building of triplets phrases." +,10,1,0,1,0,"/hello-solr-wolrd","Hello Solr Wolrd" +,11,1,0,1,0,"/hello-test-wolrd","Hello Test Wolrd" +,12,1,0,1,0,"/test-hello-solr-world","Test Hello Solr World" +,13,1,0,1,0,"/test-hello-search-world","Test Hello Search World" +,14,1,0,1,0,"/hello-the-test-search-world","Hello the Test Search World" +,15,1,0,1,0,"/hello-world-for-phrase-searching","Hello World for phrase searching" +,16,1,0,1,0,"/about","Solr is blazing-fast and open source enterprise search platform built on Apache Lucene" +"tx_solr_indexqueue_item", +,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" +,2,1,"pages",2,"pages",1449151778,0,0,0,0,0 +,3,1,"pages",3,"pages",1449151778,0,0,0,0,0 +,4,1,"pages",4,"pages",1449151778,0,0,0,0,0 +,5,1,"pages",5,"pages",1449151778,0,0,0,0,0 +,6,1,"pages",6,"pages",1449151778,0,0,0,0,0 +,7,1,"pages",7,"pages",1449151778,0,0,0,0,0 +,8,1,"pages",8,"pages",1449151778,0,0,0,0,0 +,9,1,"pages",9,"pages",1449151778,0,0,0,0,0 +,10,1,"pages",10,"pages",1449151778,0,0,0,0,0 +,11,1,"pages",11,"pages",1449151778,0,0,0,0,0 +,12,1,"pages",12,"pages",1449151778,0,0,0,0,0 +,13,1,"pages",13,"pages",1449151778,0,0,0,0,0 +,14,1,"pages",14,"pages",1449151778,0,0,0,0,0 +,15,1,"pages",15,"pages",1449151778,0,0,0,0,0 +,16,1,"pages",16,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/Fixtures/Search/phrase_search_bigram.csv b/Tests/Integration/Fixtures/Search/phrase_search_bigram.csv index e5c3c438c7..ff8e92d430 100644 --- a/Tests/Integration/Fixtures/Search/phrase_search_bigram.csv +++ b/Tests/Integration/Fixtures/Search/phrase_search_bigram.csv @@ -1,68 +1,38 @@ "pages",,,,,,, ,"uid","pid","is_siteroot","doktype","hidden","slug","title" # 100% matching -,1,0,1,1,0,"/","Bigram Phrase Search" +,2,1,1,1,0,"/","Bigram Phrase Search" # bigram phrase slop = 0 -,2,1,0,1,0,"/dump-sentence","This is some dump sentence to match Bigram Phrase" -,3,1,0,1,0,"/match-phrase-search","This is some dump sentence to match Phrase Search" +,3,1,0,1,0,"/dump-sentence","This is some dump sentence to match Bigram Phrase" +,4,1,0,1,0,"/match-phrase-search","This is some dump sentence to match Phrase Search" # bigram phrase slop = 1 -,4,1,0,1,0,"/slop-1-bigram-gag-phrase","This is some dump sentence to match slop 1 Bigram gag Phrase" -,5,1,0,1,0,"/slop-1-phrase-gag-search","This is some dump sentence to match slop 1 Phrase gag Search" +,5,1,0,1,0,"/slop-1-bigram-gag-phrase","This is some dump sentence to match slop 1 Bigram gag Phrase" +,6,1,0,1,0,"/slop-1-phrase-gag-search","This is some dump sentence to match slop 1 Phrase gag Search" # bigram phrase slop = 2 -,6,1,0,1,0,"/slop-2-bigram-gag-gag-phrase","This is some dump sentence to match slop 2 Bigram gag gag Phrase" -,7,1,0,1,0,"/slop-2-phrase-gag-gag-search","This is some dump sentence to match slop 2 Phrase gag gag Search" -,8,1,0,1,0,"/no-docs-match","This is some dump sentence to match no docs" -,9,1,0,1,0,"/no-match","This is some dump sentence to match nothing" -,10,1,0,1,0,"/bphrase-disabled","To disable BPhrase set it to 0." -,11,1,0,1,0,"/match-phrase-only","This is some dump sentence to match Phrase only" -,12,1,0,1,0,"/match-search-only","This is some dump sentence to match Search only" -,13,1,0,1,0,"/match-bigram-only","This is some dump sentence to match Bigram only" -,14,1,0,1,0,"/implicit-bigram-phrase-pf2","Implicit Bigram gag gag gag gag Phrase gag gag gag gag searches in pf2 fields." -,15,1,0,1,0,"/bigram-gag-gag-gag-gag-phrase","Bigram gag gag gag gag Phrase gag gag gag gag Search is fascinating feature." -"sys_template",,,,,,, -,"uid","pid","root","clear","sorting","constants","config", -,1,1,1,3,100,""," -page = PAGE -page.typeNum = 0 - -plugin.tx_solr { - search.query.bigramPhrase = 1 - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - - } - } -}" +,7,1,0,1,0,"/slop-2-bigram-gag-gag-phrase","This is some dump sentence to match slop 2 Bigram gag gag Phrase" +,8,1,0,1,0,"/slop-2-phrase-gag-gag-search","This is some dump sentence to match slop 2 Phrase gag gag Search" +,9,1,0,1,0,"/no-docs-match","This is some dump sentence to match no docs" +,10,1,0,1,0,"/no-match","This is some dump sentence to match nothing" +,11,1,0,1,0,"/bphrase-disabled","To disable BPhrase set it to 0." +,12,1,0,1,0,"/match-phrase-only","This is some dump sentence to match Phrase only" +,13,1,0,1,0,"/match-search-only","This is some dump sentence to match Search only" +,14,1,0,1,0,"/match-bigram-only","This is some dump sentence to match Bigram only" +,15,1,0,1,0,"/implicit-bigram-phrase-pf2","Implicit Bigram gag gag gag gag Phrase gag gag gag gag searches in pf2 fields." +,16,1,0,1,0,"/bigram-gag-gag-gag-gag-phrase","Bigram gag gag gag gag Phrase gag gag gag gag Search is fascinating feature." +"tx_solr_indexqueue_item", +,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" +,2,1,"pages",2,"pages",1449151778,0,0,0,0,0 +,3,1,"pages",3,"pages",1449151778,0,0,0,0,0 +,4,1,"pages",4,"pages",1449151778,0,0,0,0,0 +,5,1,"pages",5,"pages",1449151778,0,0,0,0,0 +,6,1,"pages",6,"pages",1449151778,0,0,0,0,0 +,7,1,"pages",7,"pages",1449151778,0,0,0,0,0 +,8,1,"pages",8,"pages",1449151778,0,0,0,0,0 +,9,1,"pages",9,"pages",1449151778,0,0,0,0,0 +,10,1,"pages",10,"pages",1449151778,0,0,0,0,0 +,11,1,"pages",11,"pages",1449151778,0,0,0,0,0 +,12,1,"pages",12,"pages",1449151778,0,0,0,0,0 +,13,1,"pages",13,"pages",1449151778,0,0,0,0,0 +,14,1,"pages",14,"pages",1449151778,0,0,0,0,0 +,15,1,"pages",15,"pages",1449151778,0,0,0,0,0 +,16,1,"pages",16,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/Fixtures/Search/phrase_search_trigram.csv b/Tests/Integration/Fixtures/Search/phrase_search_trigram.csv index 85049b02fc..64d6ee0927 100644 --- a/Tests/Integration/Fixtures/Search/phrase_search_trigram.csv +++ b/Tests/Integration/Fixtures/Search/phrase_search_trigram.csv @@ -1,67 +1,38 @@ "pages",,,,,,, ,"uid","pid","is_siteroot","doktype","hidden","slug","title" # 100% matching -,1,0,1,1,0,"/","Awesome Trigram Phrase Search" +,2,1,1,1,0,"/","Awesome Trigram Phrase Search" # trigram phrase slop = 0 -,2,1,0,1,0,"/dump-sentence","This is some dump sentence to match Awesome Trigram Phrase" -,3,1,0,1,0,"/match-trigram-phrase-search","This is some dump sentence to match Trigram Phrase Search" +,3,1,0,1,0,"/dump-sentence","This is some dump sentence to match Awesome Trigram Phrase" +,4,1,0,1,0,"/match-trigram-phrase-search","This is some dump sentence to match Trigram Phrase Search" # trigram phrase slop = 1 -,4,1,0,1,0,"/slop-1-trigram-phrase","This is some dump sentence to match slop 1 Awesome gag Trigram Phrase" -,5,1,0,1,0,"/slop-1-trigram-phrase-gag-search","This is some dump sentence to match slop 1 Trigram Phrase gag Search" +,5,1,0,1,0,"/slop-1-trigram-phrase","This is some dump sentence to match slop 1 Awesome gag Trigram Phrase" +,6,1,0,1,0,"/slop-1-trigram-phrase-gag-search","This is some dump sentence to match slop 1 Trigram Phrase gag Search" # trigram phrase slop = 2 -,6,1,0,1,0,"/slop-2-trigram-gag-gag-phrase","This is some dump sentence to match slop 2 Awesome gag gag Trigram Phrase" -,7,1,0,1,0,"/slop-2-phrase-gag-gag-search","This is some dump sentence to match slop 2 Trigram Phrase gag gag Search" -,8,1,0,1,0,"/no-docs-match","This is some dump sentence to match no docs" -,9,1,0,1,0,"/no-match","This is some dump sentence to match nothing" -,10,1,0,1,0,"/bphrase-disabled","To disable BPhrase set it to 0." -,11,1,0,1,0,"/match-phrase-only","This is some dump sentence to match Phrase only" -,12,1,0,1,0,"/match-search-only","This is some dump sentence to match Search only" -,13,1,0,1,0,"/match-trigram-only","This is some dump sentence to match Trigram only" -,14,1,0,1,0,"/match-awesome-only","This is some dump sentence to match Awesome only" -,15,1,0,1,0,"/trigram-gag-gag-gag-gag-phrase","Trigram gag gag gag gag Phrase gag gag gag gag Search is fascinating feature." -"sys_template",,,,,,, -,"uid","pid","root","clear","sorting","constants","config", -,1,1,1,3,100,""," -page = PAGE -page.typeNum = 0 - -plugin.tx_solr { - search.query.trigramPhrase = 1 - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - } - } -}" +,7,1,0,1,0,"/slop-2-trigram-gag-gag-phrase","This is some dump sentence to match slop 2 Awesome gag gag Trigram Phrase" +,8,1,0,1,0,"/slop-2-phrase-gag-gag-search","This is some dump sentence to match slop 2 Trigram Phrase gag gag Search" +,9,1,0,1,0,"/no-docs-match","This is some dump sentence to match no docs" +,10,1,0,1,0,"/no-match","This is some dump sentence to match nothing" +,11,1,0,1,0,"/bphrase-disabled","To disable BPhrase set it to 0." +,12,1,0,1,0,"/match-phrase-only","This is some dump sentence to match Phrase only" +,13,1,0,1,0,"/match-search-only","This is some dump sentence to match Search only" +,14,1,0,1,0,"/match-trigram-only","This is some dump sentence to match Trigram only" +,15,1,0,1,0,"/match-awesome-only","This is some dump sentence to match Awesome only" +,16,1,0,1,0,"/trigram-gag-gag-gag-gag-phrase","Trigram gag gag gag gag Phrase gag gag gag gag Search is fascinating feature." +"tx_solr_indexqueue_item", +,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" +,2,1,"pages",2,"pages",1449151778,0,0,0,0,0 +,3,1,"pages",3,"pages",1449151778,0,0,0,0,0 +,4,1,"pages",4,"pages",1449151778,0,0,0,0,0 +,5,1,"pages",5,"pages",1449151778,0,0,0,0,0 +,6,1,"pages",6,"pages",1449151778,0,0,0,0,0 +,7,1,"pages",7,"pages",1449151778,0,0,0,0,0 +,8,1,"pages",8,"pages",1449151778,0,0,0,0,0 +,9,1,"pages",9,"pages",1449151778,0,0,0,0,0 +,10,1,"pages",10,"pages",1449151778,0,0,0,0,0 +,11,1,"pages",11,"pages",1449151778,0,0,0,0,0 +,12,1,"pages",12,"pages",1449151778,0,0,0,0,0 +,13,1,"pages",13,"pages",1449151778,0,0,0,0,0 +,14,1,"pages",14,"pages",1449151778,0,0,0,0,0 +,15,1,"pages",15,"pages",1449151778,0,0,0,0,0 +,16,1,"pages",16,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_custom_pagetype_into_solr.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_custom_pagetype_into_solr.csv index bc6f0243d2..16d0ad552a 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_custom_pagetype_into_solr.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_custom_pagetype_into_solr.csv @@ -1,62 +1,6 @@ "pages", ,"uid","pid","is_siteroot","doktype","slug","title","subtitle","crdate","tstamp" -,1,0,1,1,"/","hello solr","the subtitle",1449151778,1449151778 +,2,1,1,1,"/hello_solr","hello solr","the subtitle",1449151778,1449151778 "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" -,4711,1,"pages",1,"mytype",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," -page = PAGE -page.typeNum = 0 -config.index_enable = 1 - -plugin.tx_solr { - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - custom_stringS = TEXT - custom_stringS.value = my text - } - } - - mytype < pages - mytype { - allowedPageTypes = 130 - additionalWhereClause = doktype = 130 - fields { - custom_stringS = TEXT - custom_stringS.value = my text from custom page type - } - } - } - } -}" +,4711,1,"pages",2,"mytype",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_into_solr.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_into_solr.csv index f248b2d217..4ff064bf89 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_into_solr.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_into_solr.csv @@ -1,53 +1,6 @@ "pages", ,"uid","pid","is_siteroot","doktype","slug","title","subtitle","crdate","tstamp" -,1,0,1,1,"/","hello solr","the subtitle",1449151778,1449151778 +,2,1,1,1,"/hello_solr","hello solr","the subtitle",1449151778,1449151778 "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" -,4711,1,"pages",1,"pages",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," -page = PAGE -page.typeNum = 0 - -config.index_enable = 1 - -plugin.tx_solr { - enabled = 1 - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - custom_stringS = TEXT - custom_stringS.value = my text - } - } - - } - } -}" +,4711,1,"pages",2,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_mounted_page.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_mounted_page.csv index 0ddc461f82..93b85faada 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_mounted_page.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_mounted_page.csv @@ -12,7 +12,6 @@ "pages", ,"uid","pid","is_siteroot","doktype","mount_pid","mount_pid_ol","slug","title" # Site tree -,1,0,1,1,0,0,"/","Page (Root)" ,14,1,0,7,24,1,"/mount-point","Mount Point" # Shared Pages tree ,20,0,0,254,0,0,"/","Shared-Pages" @@ -22,48 +21,4 @@ ,99,24,0,"text","Some Lorem Ipsum conteint!" "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" -,4711,1,"pages",24,"pages",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," -page = PAGE -page.typeNum = 0 -config.index_enable = 1 - -plugin.tx_solr { - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - } - } -}" +,4711,1,"pages",24,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_multiple_mounted_page.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_multiple_mounted_page.csv index b2cc6eccc3..c97e1be9a2 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_multiple_mounted_page.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_multiple_mounted_page.csv @@ -15,11 +15,8 @@ "pages", ,"uid","pid","is_siteroot","doktype","mount_pid","mount_pid_ol","slug","title" # Site tree a -,1,0,1,1,0,0,"/","Page (Root a)" ,14,1,0,7,44,1,"/mount-point","Mount Point" ,24,1,0,7,44,1,"/mount-point-2","Mount Point 2" -# Site tree b -,2,0,1,1,0,0,"/","Page (Root b)" # Shared Pages tree ,20,0,0,254,0,0,"/shared-pages","Shared-Pages" ,44,20,0,1,0,0,"/first-shared","FirstShared (Not root)" @@ -30,93 +27,3 @@ ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" ,4711,1,"pages",44,"pages",1449151778,0,0,0,0,0 ,4712,1,"pages",44,"pages",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," -page = PAGE -page.typeNum = 0 -config.index_enable = 1 - -plugin.tx_solr { - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - - } - } -}" -,2,2,1,3,100," -page = PAGE -page.typeNum = 0 - -plugin.tx_solr { - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - - } - } -}" diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_category.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_category.csv index 50c71a1fda..24e220bf84 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_category.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_category.csv @@ -1,6 +1,5 @@ "pages", ,"uid","pid","is_siteroot","doktype","slug","title" -,1,0,1,1,"/","Page" ,10,1,0,1,"/sub-page","Sub page" "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" @@ -10,54 +9,4 @@ ,123,0,"Test" "sys_category_record_mm", ,"uid_local","uid_foreign","tablenames","fieldname","sorting","sorting_foreign" -,123,10,"pages","categories",1,1 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," - -page = PAGE -page.typeNum = 0 -config.index_enable = 1 - -# very simple rendering -page.10 = CONTENT -page.10 { - table = tt_content - select.orderBy = sorting - select.where = colPos=0 - renderObj = COA - renderObj { - 10 = TEXT - 10.field = bodytext - } -} - -plugin.tx_solr { - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - pageHierarchy_stringM = pathToHierarchy - } - - queue { - pages = 1 - pages { - table = pages - fields { - title = title - categories_stringM = SOLR_RELATION - categories_stringM { - localField = categories - foreignLabelField = title - multiValue = 1 - } - } - } - } - } -}" +,123,10,"pages","categories",1,1 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_page.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_page.csv index e3f25fafb1..78ec629b7c 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_page.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_page_with_relation_to_page.csv @@ -1,64 +1,12 @@ "pages", ,"uid","pid","is_siteroot","doktype","sys_language_uid","l10n_parent","hidden","deleted","slug","title","relations" -,1,0,1,1,0,0,0,0,"/","Page",0 -,2,0,1,1,1,1,0,0,"/","Seite",1 +,2,1,1,1,0,0,0,0,"/page","Page",0 +,3,1,1,1,1,2,0,0,"/seite","Seite",1 ,10,1,0,1,0,0,0,0,"/related-page","Related page",1 ,11,1,0,1,1,10,0,0,"/verwandte-seite","Verwandte Seite",1 "tx_fakeextension3_pages_mm", ,"uid_local","uid_foreign","tablenames","fieldname","sorting","sorting_foreign" -,10,2,"pages","page_relations",0,0 +,10,3,"pages","page_relations",0,0 "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" -,4711,1,"pages",1,"pages",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," - -config.index_enable = 1 - -page = PAGE -page.typeNum = 0 - -# very simple rendering -page.10 = CONTENT -page.10 { - table = tt_content - select.orderBy = sorting - select.where = colPos=0 - renderObj = COA - renderObj { - 10 = TEXT - 10.field = bodytext - } -} - -plugin.tx_solr { - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - pageHierarchy_stringM = pathToHierarchy - } - - queue { - pages = 1 - pages { - table = pages - fields { - title = title - relatedPageTitles_stringM = SOLR_RELATION - relatedPageTitles_stringM { - localField = page_relations - enableRecursiveValueResolution = 0 - multiValue = 1 - } - } - } - } - } -}" - +,4711,1,"pages",2,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_with_additional_fields_into_solr.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_with_additional_fields_into_solr.csv index 29c2256c6f..60b4e2b293 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_with_additional_fields_into_solr.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_index_with_additional_fields_into_solr.csv @@ -1,56 +1,6 @@ "pages", ,"uid","pid","is_siteroot","doktype","slug","title","subtitle","crdate","tstamp" -,1,0,1,1,"/","hello solr","the subtitle",1449151778,1449151778 +,2,1,1,1,"/","hello solr","the subtitle",1449151778,1449151778 "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" -,4711,1,"pages",1,"pages",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," -page = PAGE -page.typeNum = 0 -config.index_enable = 1 - -plugin.tx_solr { - enabled = 1 - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - additionalFields { - additional_sortSubTitle_stringS = subtitle - additional_custom_stringS = TEXT - additional_custom_stringS.value = my text - } - - queue { - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - } - } - } - } -}" +,4711,1,"pages",2,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_overwrite_configuration_in_rootline.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_overwrite_configuration_in_rootline.csv index 3a7659c103..a666146a7d 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_overwrite_configuration_in_rootline.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/can_overwrite_configuration_in_rootline.csv @@ -1,66 +1,6 @@ "pages", ,"uid","pid","is_siteroot","doktype","slug","title","subtitle","crdate","tstamp" -,1,0,1,1,"/","hello solr","the subtitle",1449151778,1449151778 ,2,1,0,1,"/hello-subpage","hello subpage","the subpage subtitle",1449151778,1449151778 "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" -,4711,1,"pages",2,"pages",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," -page = PAGE -page.typeNum = 0 -config.index_enable = 1 - -plugin.tx_solr { - enabled = 1 - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - custom_stringS = TEXT - custom_stringS.value = my text - } - } - - } - } -}" -,2,2,0,0,100," -plugin.tx_solr { - index { - queue { - pages { - fields { - additional_stringS = TEXT - additional_stringS.value = from rootline - } - } - } - } -}" +,4711,1,"pages",2,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/does_not_die_if_page_not_available.csv b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/does_not_die_if_page_not_available.csv index eb0dc15cf2..201435dac6 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/does_not_die_if_page_not_available.csv +++ b/Tests/Integration/IndexQueue/FrontendHelper/Fixtures/does_not_die_if_page_not_available.csv @@ -1,58 +1,3 @@ -"pages", -,"uid","pid","is_siteroot","doktype","slug","title","subtitle","crdate","tstamp" -,1,0,1,1,"/","hello solr","the subtitle",1449151778,1449151778 "tx_solr_indexqueue_item", ,"uid","root","item_type","item_uid","indexing_configuration","changed","indexed","has_indexing_properties","indexing_priority","indexed","errors" -,4711,1,"pages",1,"pages",1449151778,0,0,0,0,0 -"sys_template", -,"uid","pid","root","clear","sorting","config" -,1,1,1,3,100," -config.index_enable = 1 -config.sys_language_uid = 3 -config.sys_language_mode = strict - -page = PAGE -page.typeNum = 0 - -plugin.tx_solr { - enabled = 1 - - index { - fieldProcessingInstructions { - changed = timestampToIsoDate - created = timestampToIsoDate - endtime = timestampToUtcIsoDate - rootline = pageUidToHierarchy - } - - queue { - - // mapping tableName.fields.SolrFieldName => TableFieldName (+ cObj processing) - - pages = 1 - pages { - initialization = ApacheSolrForTypo3\Solr\IndexQueue\Initializer\Page - - // allowed page types (doktype) when indexing records from pages - allowedPageTypes = 1,7 - - indexingPriority = 0 - - indexer = ApacheSolrForTypo3\Solr\IndexQueue\PageIndexer - indexer { - // add options for the indexer here - } - - // Only index standard pages and mount points that are not overlayed. - additionalWhereClause = (doktype = 1 OR (doktype=7 AND mount_pid_ol=0)) AND no_search = 0 - - fields { - sortSubTitle_stringS = subtitle - custom_stringS = TEXT - custom_stringS.value = my text - } - } - - } - } -}" +,4711,999,"pages",1636120156,"pages",1449151778,0,0,0,0,0 \ No newline at end of file diff --git a/Tests/Integration/IndexQueue/FrontendHelper/PageIndexerTest.php b/Tests/Integration/IndexQueue/FrontendHelper/PageIndexerTest.php index dacca4bbb5..a669bd8688 100644 --- a/Tests/Integration/IndexQueue/FrontendHelper/PageIndexerTest.php +++ b/Tests/Integration/IndexQueue/FrontendHelper/PageIndexerTest.php @@ -19,7 +19,6 @@ use ApacheSolrForTypo3\Solr\Tests\Integration\IntegrationTest; use Psr\Http\Message\ResponseInterface; -use TYPO3\CMS\Core\Exception\SiteNotFoundException; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Frontend\Page\CacheHashCalculator; @@ -36,12 +35,6 @@ class PageIndexerTest extends IntegrationTest '../vendor/apache-solr-for-typo3/solr/Tests/Integration/Fixtures/Extensions/fake_extension3', ]; - /** - * @inheritdoc - * @todo: Remove unnecessary fixtures and remove that property as intended. - */ - protected bool $skipImportRootPagesAndTemplatesForConfiguredSites = true; - protected function setUp(): void { parent::setUp(); @@ -60,13 +53,24 @@ protected function tearDown(): void /** * @test */ - public function canIndexPageIntoSolr() + public function canIndexPageIntoSolr(): void { $this->cleanUpSolrServerAndAssertEmpty(); $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_into_solr.csv'); + $this->addSimpleFrontendRenderingToTypoScriptRendering( + 1, + /* @lang TYPO3_TypoScript */ + ' + plugin.tx_solr.index.queue.pages.fields { + sortSubTitle_stringS = subtitle + custom_stringS = TEXT + custom_stringS.value = my text + } + ' + ); - $this->indexQueuedPage(); + $this->indexQueuedPage(2); // we wait to make sure the document will be available in solr $this->waitToBeVisibleInSolr(); @@ -81,13 +85,28 @@ public function canIndexPageIntoSolr() /** * @test */ - public function canIndexPageWithCustomPageTypeIntoSolr() + public function canIndexPageWithCustomPageTypeIntoSolr(): void { $this->cleanUpSolrServerAndAssertEmpty(); $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_custom_pagetype_into_solr.csv'); - $this->indexQueuedPage(); + // @TODO: Check page type in fixture, currently not set to 130 + $this->addSimpleFrontendRenderingToTypoScriptRendering( + 1, + /* @lang TYPO3_TypoScript */ + ' + plugin.tx_solr.index.queue.mytype < plugin.tx_solr.index.queue.pages + plugin.tx_solr.index.queue.mytype { + allowedPageTypes = 130 + additionalWhereClause = doktype = 130 + fields.custom_stringS = TEXT + fields.custom_stringS.value = my text from custom page type + } + ' + ); + + $this->indexQueuedPage(2); // we wait to make sure the document will be available in solr $this->waitToBeVisibleInSolr(); @@ -103,15 +122,27 @@ public function canIndexPageWithCustomPageTypeIntoSolr() * * @test */ - public function canIndexTranslatedPageToPageRelation() + public function canIndexTranslatedPageToPageRelation(): void { $this->cleanUpSolrServerAndAssertEmpty('core_en'); $this->cleanUpSolrServerAndAssertEmpty('core_de'); $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_page_with_relation_to_page.csv'); + $this->addSimpleFrontendRenderingToTypoScriptRendering( + 1, + /* @lang TYPO3_TypoScript */ + ' + plugin.tx_solr.index.queue.pages.fields.relatedPageTitles_stringM = SOLR_RELATION + plugin.tx_solr.index.queue.pages.fields.relatedPageTitles_stringM { + localField = page_relations + enableRecursiveValueResolution = 0 + multiValue = 1 + } + ' + ); - $this->indexQueuedPage(1, '/en/'); - $this->indexQueuedPage(1, '/de/'); + $this->indexQueuedPage(2, '/en/'); + $this->indexQueuedPage(2, '/de/'); // do we have the record in the index with the value from the mm relation? $this->waitToBeVisibleInSolr('core_en'); @@ -134,11 +165,24 @@ public function canIndexTranslatedPageToPageRelation() * * @test */ - public function canIndexPageToCategoryRelation() + public function canIndexPageToCategoryRelation(): void { $this->cleanUpSolrServerAndAssertEmpty('core_en'); $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_page_with_relation_to_category.csv'); + $this->addSimpleFrontendRenderingToTypoScriptRendering( + 1, + /* @lang TYPO3_TypoScript */ + ' + plugin.tx_solr.index.queue.pages.fields.categories_stringM = SOLR_RELATION + plugin.tx_solr.index.queue.pages.fields.categories_stringM { + localField = categories + foreignLabelField = title + multiValue = 1 + } + ' + ); + $this->indexQueuedPage(10); $this->waitToBeVisibleInSolr('core_en'); @@ -153,10 +197,21 @@ public function canIndexPageToCategoryRelation() /** * @test */ - public function canIndexPageIntoSolrWithAdditionalFields() + public function canIndexPageIntoSolrWithAdditionalFields(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_with_additional_fields_into_solr.csv'); - $this->indexQueuedPage(); + $this->addSimpleFrontendRenderingToTypoScriptRendering( + 1, + /* @lang TYPO3_TypoScript */ + ' + plugin.tx_solr.index.additionalFields { + additional_sortSubTitle_stringS = subtitle + additional_custom_stringS = TEXT + additional_custom_stringS.value = my text + } + ' + ); + $this->indexQueuedPage(2); // we wait to make sure the document will be available in solr $this->waitToBeVisibleInSolr(); @@ -176,9 +231,18 @@ public function canIndexPageIntoSolrWithAdditionalFields() /** * @test */ - public function canIndexPageIntoSolrWithAdditionalFieldsFromRootLine() + public function canIndexPageIntoSolrWithAdditionalFieldsFromRootLine(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/can_overwrite_configuration_in_rootline.csv'); + $this->addSimpleFrontendRenderingToTypoScriptRendering( + 1, + /* @lang TYPO3_TypoScript */ + ' + plugin.tx_solr.index.queue.pages.fields.additional_stringS = TEXT + plugin.tx_solr.index.queue.pages.fields.additional_stringS.value = from rootline + ' + ); + $this->indexQueuedPage(2); // we wait to make sure the document will be available in solr @@ -195,10 +259,13 @@ public function canIndexPageIntoSolrWithAdditionalFieldsFromRootLine() /** * @test */ - public function canExecutePostProcessor() + public function canExecutePostProcessor(): void { + $this->cleanUpSolrServerAndAssertEmpty(); + $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_into_solr.csv'); - $this->indexQueuedPage(); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); + $this->indexQueuedPage(2); // we wait to make sure the document will be available in solr $this->waitToBeVisibleInSolr(); @@ -211,10 +278,20 @@ public function canExecutePostProcessor() /** * @test */ - public function canExecuteAdditionalPageIndexer() + public function canExecuteAdditionalPageIndexer(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_into_solr.csv'); - $this->indexQueuedPage(1, '/en/', ['additionalTestPageIndexer' => true]); + $this->addSimpleFrontendRenderingToTypoScriptRendering( + 1, + /* @lang TYPO3_TypoScript */ + ' + plugin.tx_solr.index.queue.pages.fields { + custom_stringS = TEXT + custom_stringS.value = my text + } + ' + ); + $this->indexQueuedPage(2, '/en/', ['additionalTestPageIndexer' => true]); // we wait to make sure the document will be available in solr $this->waitToBeVisibleInSolr(); @@ -242,12 +319,13 @@ public function canExecuteAdditionalPageIndexer() * * @test */ - public function canIndexMountedPage() + public function canIndexMountedPage(): void { $GLOBALS['TYPO3_CONF_VARS']['FE']['enable_mount_pids'] = 1; $this->cleanUpSolrServerAndAssertEmpty(); $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_mounted_page.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->indexQueuedPage(24, '/en/', ['MP' => '24-14']); // we wait to make sure the document will be available in solr @@ -276,10 +354,11 @@ public function canIndexMountedPage() * ——[24] Mount Point (to [24] to show contents from) * @test */ - public function canIndexMultipleMountedPage() + public function canIndexMultipleMountedPage(): void { $this->cleanUpSolrServerAndAssertEmpty(); $this->importCSVDataSet(__DIR__ . '/Fixtures/can_index_multiple_mounted_page.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->indexQueuedPage(44, '/en/', ['MP' => '44-14']); $this->indexQueuedPage(44, '/en/', ['MP' => '44-24']); @@ -300,39 +379,16 @@ public function canIndexMultipleMountedPage() * * @test */ - public function phpProcessDoesNotDieIfPageIsNotAvailable() + public function phpProcessDoesNotDieIfPageIsNotAvailable(): void { - // @todo: 1636120156 - self::markTestIncomplete('The behaviour since TYPO3 11 and EXT:solr 11.5 must be checked twice, to be able to finalize request properly and mark items failed.'); - $this->registerShutdownFunctionToPrintExplanationOf404HandlingOnCMSIfDieIsCalled(); - $this->expectException(SiteNotFoundException::class); - $this->importCSVDataSet(__DIR__ . '/Fixtures/does_not_die_if_page_not_available.csv'); - $this->indexQueuedPage(1636120156); - } - - /** - * Registers shutdown function to print proper information about TYPO3 CMS behaviour on unavailable pages. - */ - protected function registerShutdownFunctionToPrintExplanationOf404HandlingOnCMSIfDieIsCalled() - { - register_shutdown_function(function () { - // prompt only after phpProcessDoesNotDieIfPageIsNotAvailable() test case - if ($this->getName() !== 'phpProcessDoesNotDieIfPageIsNotAvailable') { - return; - } - - // don't show HTML or other stuff from CMS in output - ob_clean(); + $response = $this->indexQueuedPage(1636120156); - $message = PHP_EOL . PHP_EOL . PHP_EOL . - 'Note: This test case kills whole PHPUnit process on failing, which is expected behaviour, because TYPO3 CMS uses die() function in cases if page or record is not available.'; - $message .= PHP_EOL . PHP_EOL . 'TYPO3 CMS API for registering shutdown callbacks for handling of unavailable pages is changed.' . PHP_EOL . - 'TypoScriptFrontendController::pageUnavailableAndExit() is not called anymore.' . PHP_EOL . - 'Please refer to the TYPO3 CMS documentation and use new API for this functionality.'; + $decodedResponse = json_decode($response->getBody()->getContents(), true); + self::assertIsArray($decodedResponse, 'Response couldn\'t be decoded'); - printf(PHP_EOL . PHP_EOL . "\033[01;31m%s\033[0m", $message); - }); + $actionResults = unserialize($decodedResponse['actionResults']['indexPage']); + self::assertFalse($actionResults['pageIndexed'] ?? null, 'Index page result not set to false as expected!'); } /** diff --git a/Tests/Integration/SearchTest.php b/Tests/Integration/SearchTest.php index 5fcd5609f4..f1c00a6b17 100644 --- a/Tests/Integration/SearchTest.php +++ b/Tests/Integration/SearchTest.php @@ -23,12 +23,14 @@ use ApacheSolrForTypo3\Solr\Domain\Search\Query\Query; use ApacheSolrForTypo3\Solr\Domain\Search\Query\QueryBuilder; use ApacheSolrForTypo3\Solr\IndexQueue\FrontendHelper\PageIndexer; -use ApacheSolrForTypo3\Solr\IndexQueue\Item; +use ApacheSolrForTypo3\Solr\IndexQueue\PageIndexerRequest; use ApacheSolrForTypo3\Solr\Search; use ApacheSolrForTypo3\Solr\System\Configuration\ConfigurationManager; use ApacheSolrForTypo3\Solr\System\Configuration\TypoScriptConfiguration; +use TYPO3\CMS\Core\Http\ServerRequest; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Frontend\Controller\TypoScriptFrontendController; +use TYPO3\CMS\Frontend\Event\AfterCacheableContentIsGeneratedEvent; /** * Test class to perform a search on a real solr server @@ -37,11 +39,6 @@ */ class SearchTest extends IntegrationTest { - /** - * @inheritdoc - * @todo: Remove unnecessary fixtures and remove that property as intended. - */ - protected bool $skipImportRootPagesAndTemplatesForConfiguredSites = true; protected QueryBuilder $queryBuilder; protected function setUp(): void @@ -62,19 +59,13 @@ protected function tearDown(): void */ public function canSearchForADocument(): void { + $this->cleanUpSolrServerAndAssertEmpty(); $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/can_search.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); - $tsfe = $this->getConfiguredTSFE(); - $pageIndexer = GeneralUtility::makeInstance(PageIndexer::class); - $indexQueueItem = new Item(['root' => 1, 'indexing_configuration' => 'pages', 'item_type' => 'pages']); - $indexQueueItem->setRecord(['uid' => 1]); - $pageIndexer->setupConfiguration(); - $pageIndexer->index($indexQueueItem, $tsfe); - - $this->waitToBeVisibleInSolr(); + $this->fillIndexForPhraseSearchTests(2, 2); $searchInstance = GeneralUtility::makeInstance(Search::class); - $query = $this->queryBuilder ->newSearchQuery('hello') ->useQueryFields(QueryFields::fromString('content^40.0, title^5.0, keywords^2.0, tagsH1^5.0, tagsH2H3^3.0, tagsH4H5H6^2.0, tagsInline^1.0, description^4.0, abstract^1.0, subtitle^1.0, navtitle^1.0, author^1.0')) @@ -92,6 +83,7 @@ public function canSearchForADocument(): void public function canHighlightTerms(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/phrase_search.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->fillIndexForPhraseSearchTests(); $searchInstance = GeneralUtility::makeInstance(Search::class); @@ -158,6 +150,7 @@ public function canHighlightTerms(): void public function implicitPhraseSearchingBoostsDocsWithOccurringPhrase(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/phrase_search.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->fillIndexForPhraseSearchTests(); $searchInstance = GeneralUtility::makeInstance(Search::class); @@ -191,6 +184,7 @@ public function implicitPhraseSearchingBoostsDocsWithOccurringPhrase(): void public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByPhraseSlop(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/phrase_search.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->fillIndexForPhraseSearchTests(); $searchInstance = GeneralUtility::makeInstance(Search::class); @@ -245,13 +239,13 @@ public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByPhraseSlop() && $parsedDatasByPhraseSlop[0]->response->docs[0]->getUid() === $parsedDatasByPhraseSlop[2]->response->docs[0]->getUid(), 'Phrase search does not work properly. Solr should position the document independent from slop value at first position.'); // the slop value of 1 moves doc UID = 11 to the second position // @extensionScannerIgnoreLine - self::assertSame(11, $parsedDatasByPhraseSlop[1]->response->docs[1]->getUid(), 'Phrase slop setting does not work as expected.'); + self::assertSame(12, $parsedDatasByPhraseSlop[1]->response->docs[1]->getUid(), 'Phrase slop setting does not work as expected.'); // the slop value of 2 moves doc UID = 3 to the fifth position // @extensionScannerIgnoreLine - self::assertSame(3, $parsedDatasByPhraseSlop[2]->response->docs[4]->getUid(), 'Phrase slop setting does not work as expected. The Phrase Slop value of 2 has no influence on boosts.'); + self::assertSame(4, $parsedDatasByPhraseSlop[2]->response->docs[4]->getUid(), 'Phrase slop setting does not work as expected. The Phrase Slop value of 2 has no influence on boosts.'); // the slop value of 2 has an influence of positions up to 8 // @extensionScannerIgnoreLine - self::assertSame(2, $parsedDatasByPhraseSlop[2]->response->docs[7]->getUid(), 'Phrase slop setting does not work as expected.'); + self::assertSame(3, $parsedDatasByPhraseSlop[2]->response->docs[7]->getUid(), 'Phrase slop setting does not work as expected.'); } /** @@ -262,13 +256,14 @@ public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByPhraseSlop() public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByBigramPhraseSlop(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/phrase_search_bigram.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->fillIndexForPhraseSearchTests(); $searchInstance = GeneralUtility::makeInstance(Search::class); $this->switchPhraseSearchFeature('bigramPhrase', 1); - $query = $this->getSearchQueryForSolr(); + $this->getSearchQueryForSolr(); $this->queryBuilder->useQueryString('Bigram Phrase Search'); // Boost the document with query to make it first. @@ -317,10 +312,10 @@ public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByBigramPhrase && $parsedDatasByPhraseSlop[0]->response->docs[0]->getUid() === $parsedDatasByPhraseSlop[2]->response->docs[0]->getUid(), 'Bigram Phrase search does not work properly. Solr should position the documents independent from slop value at first position.'); // slop = 1 - // the slop value of 1 moves doc UID = 4 to the fourth(key 3) position + // the slop value of 1 moves doc UID = 5 to the fourth(key 3) position // @extensionScannerIgnoreLine - self::assertSame(4, $parsedDatasByPhraseSlop[1]->response->docs[3]->getUid(), 'Bigram phrase slop setting does not work as expected. It does not boost "sloppy phrase" docs for slop=1.'); - // the docuemnt on position 3 and 4 have same score + self::assertSame(5, $parsedDatasByPhraseSlop[1]->response->docs[3]->getUid(), 'Bigram phrase slop setting does not work as expected. It does not boost "sloppy phrase" docs for slop=1.'); + // the document on position 3 and 4 have same score self::assertTrue( // @extensionScannerIgnoreLine $parsedDatasByPhraseSlop[1]->response->docs[3]->getScore() === $parsedDatasByPhraseSlop[1]->response->docs[4]->getScore(), @@ -328,9 +323,9 @@ public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByBigramPhrase ); // slop = 2 - // the slop value of 2 moves doc UID = 6 to the sixth(key 5) position + // the slop value of 2 moves doc UID = 7 to the sixth(key 5) position // @extensionScannerIgnoreLine - self::assertSame(6, $parsedDatasByPhraseSlop[2]->response->docs[5]->getUid(), 'Trigram phrase slop setting does not work as expected. The Phrase Slop value of 2 has no influence on boosts.'); + self::assertSame(7, $parsedDatasByPhraseSlop[2]->response->docs[5]->getUid(), 'Trigram phrase slop setting does not work as expected. The Phrase Slop value of 2 has no influence on boosts.'); // the docuemnt on position 5 and 6 have same score self::assertTrue( // @extensionScannerIgnoreLine @@ -347,13 +342,14 @@ public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByBigramPhrase public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByTrigramPhraseSlop(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/phrase_search_trigram.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->fillIndexForPhraseSearchTests(); $searchInstance = GeneralUtility::makeInstance(Search::class); $this->switchPhraseSearchFeature('trigramPhrase', 1); - $query = $this->getSearchQueryForSolr(); + $this->getSearchQueryForSolr(); $this->queryBuilder ->useQueryString('Awesome Trigram Phrase Search') // Boost the document with query to make it first. @@ -427,6 +423,7 @@ public function implicitPhraseSearchSloppyPhraseBoostCanBeAdjustedByTrigramPhras public function explicitPhraseSearchMatchesMorePrecise(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/phrase_search.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->fillIndexForPhraseSearchTests(); /** @var \ApacheSolrForTypo3\Solr\Search $searchInstance */ @@ -450,6 +447,7 @@ public function explicitPhraseSearchMatchesMorePrecise(): void public function explicitPhraseSearchPrecisionCanBeAdjustedByQuerySlop(): void { $this->importCSVDataSet(__DIR__ . '/Fixtures/Search/phrase_search.csv'); + $this->addTypoScriptToTemplateRecord(1, 'config.index_enable = 1'); $this->fillIndexForPhraseSearchTests(); $searchInstance = GeneralUtility::makeInstance(Search::class); @@ -485,15 +483,20 @@ public function explicitPhraseSearchPrecisionCanBeAdjustedByQuerySlop(): void self::assertSame(7, $parsedData->response->numFound, 'Found wrong number of decuments by explicit phrase search query.'); } - protected function fillIndexForPhraseSearchTests(): void + protected function fillIndexForPhraseSearchTests(int $startUid = 2, int $endUid = 16): void { - for ($i = 1; $i <= 15; $i++) { + for ($i = $startUid; $i <= $endUid; $i++) { $tsfe = $this->getConfiguredTSFE($i); + + $serverRequest = new ServerRequest(); + $pageIndexerRequest = GeneralUtility::makeInstance(PageIndexerRequest::class); + $pageIndexerRequest->setParameter('item', $i); + $serverRequest = $serverRequest->withAttribute('solr.pageIndexingInstructions', $pageIndexerRequest); + $event = new AfterCacheableContentIsGeneratedEvent($serverRequest, $tsfe, 'cache-identifier', true); + $pageIndexer = GeneralUtility::makeInstance(PageIndexer::class); - $indexQueueItem = new Item(['root' => 1, 'indexing_configuration' => 'pages', 'item_type' => 'pages']); - $indexQueueItem->setRecord(['uid' => $i]); - $pageIndexer->setupConfiguration(); - $pageIndexer->index($indexQueueItem, $tsfe); + $pageIndexer->activate(); + $pageIndexer($event); } $this->waitToBeVisibleInSolr(); }