From 87531c2f45542b804f642625f61935864baa327b Mon Sep 17 00:00:00 2001 From: Paul Heinze Date: Thu, 10 Aug 2023 22:04:22 +0000 Subject: [PATCH] Remove `org.apache.jena` from components (#296) * refactor main process method to separate methods * add checks for getSparqlInswertQuery() to tests * add test for getSparqlInsertQuery * remove org.apache.jena dependency from pom.xml * remove org.apache.jena dependency from pom.xml * add test for methods using org.apache.jena * refactor process method --- qanary-component-NED-DBpediaSpotlight/pom.xml | 30 +---- .../dbpediaspotlight/ned/Application.java | 17 +-- .../ned/DBpediaSpotlightNED.java | 35 +++-- .../ned/DBpediaSpotlightServiceFetcher.java | 60 ++++++--- .../DBpediaSpotlightServiceFetcherTest.java | 84 ++++++++---- qanary-component-NERD-SMAPH/pom.xml | 18 +-- qanary-component-QBE-QAnswer/pom.xml | 18 +-- qanary-component-QE-SparqlExecuter/pom.xml | 20 +-- .../qanary/sparqlexecuter/SparqlExecuter.java | 122 ++++++++++++------ .../QanaryServiceControllerTest.java | 80 ------------ .../sparqlexecuter/SparqlExecuterTest.java | 114 ++++++++++++++++ 11 files changed, 321 insertions(+), 277 deletions(-) delete mode 100644 qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/QanaryServiceControllerTest.java create mode 100644 qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuterTest.java diff --git a/qanary-component-NED-DBpediaSpotlight/pom.xml b/qanary-component-NED-DBpediaSpotlight/pom.xml index 478c54606..914f51d6e 100644 --- a/qanary-component-NED-DBpediaSpotlight/pom.xml +++ b/qanary-component-NED-DBpediaSpotlight/pom.xml @@ -4,7 +4,7 @@ 4.0.0 eu.wdaqua.qanary.component qanary-component-NED-DBpediaSpotlight - 3.3.1 + 3.4.0 org.springframework.boot spring-boot-starter-parent @@ -86,34 +86,6 @@ spring-test test - - org.apache.jena - apache-jena-libs - pom - 3.0.0 - - - org.slf4j - slf4j-log4j12 - - - org.apache.logging.log4j - log4j-to-slf4j - - - org.apache.logging.log4j - log4j-api - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.logging.log4j diff --git a/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/Application.java b/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/Application.java index 5ace6cb89..39f0ac0f9 100644 --- a/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/Application.java +++ b/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/Application.java @@ -1,9 +1,5 @@ package eu.wdaqua.qanary.component.dbpediaspotlight.ned; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -16,8 +12,6 @@ import org.springframework.web.client.RestTemplate; import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse; -import eu.wdaqua.qanary.communications.RestTemplateWithCaching; -import eu.wdaqua.qanary.component.QanaryComponent; import eu.wdaqua.qanary.component.dbpediaspotlight.ned.exceptions.DBpediaSpotlightServiceNotAvailable; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.info.Info; @@ -46,6 +40,7 @@ public static void main(String[] args) { @Bean public DBpediaSpotlightConfiguration myDBpediaSpotlightConfiguration( // + @Autowired DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher, // @Value("${dbpediaspotlight.test-question}") String testQuestion, // @Value("${dbpediaspotlight.confidence.minimum}") float confidenceMinimum, // @Value("${dbpediaspotlight.endpoint:https://api.dbpedia-spotlight.org/en/annotate}") String endpoint, // @@ -53,15 +48,10 @@ public DBpediaSpotlightConfiguration myDBpediaSpotlightConfiguration( // @Value("${dbpediaspotlight.endpoint.ssl.certificatevalidation.ignore:false}") final boolean ignore ) throws DBpediaSpotlightServiceNotAvailable { this.checkSpotlightServiceAvailability(testQuestion, endpoint, confidenceMinimum, - myDBpediaSpotlightServiceFetcher(), performLiveCheckOnComponentStart, ignore); + myDBpediaSpotlightServiceFetcher, performLiveCheckOnComponentStart, ignore); return new DBpediaSpotlightConfiguration(confidenceMinimum, endpoint); } - @Bean - public DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher() { - return new DBpediaSpotlightServiceFetcher(); - } - private void checkSpotlightServiceAvailability(String testQuestion, String endpoint, float confidenceMinimum, DBpediaSpotlightServiceFetcher dBpediaSpotlightServiceFetcher, boolean performLiveCheckOnComponentStart, boolean ignore) throws DBpediaSpotlightServiceNotAvailable { @@ -79,8 +69,7 @@ private void checkSpotlightServiceAvailability(String testQuestion, String endpo restTemplate.setRequestFactory(DBpediaSpotlightNED.getRequestFactoryForSslVerficationDeactivation()); logger.warn("SSL certificate validation deactivated."); } - dBpediaSpotlightServiceFetcher.getJsonFromService(testQuestion, endpoint, confidenceMinimum, - restTemplate, myCacheOfResponses); + dBpediaSpotlightServiceFetcher.getJsonFromService(testQuestion); return; } catch (Exception e) { err = e.toString() + " " + e.getLocalizedMessage(); diff --git a/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightNED.java b/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightNED.java index 3a2ce431d..34bad25be 100644 --- a/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightNED.java +++ b/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightNED.java @@ -1,6 +1,7 @@ package eu.wdaqua.qanary.component.dbpediaspotlight.ned; import java.io.IOException; +import java.net.URISyntaxException; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -10,8 +11,6 @@ import javax.inject.Inject; import javax.net.ssl.SSLContext; -import eu.wdaqua.qanary.commons.triplestoreconnectors.QanaryTripleStoreConnector; - import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.impl.client.CloseableHttpClient; @@ -29,13 +28,15 @@ import com.google.gson.JsonArray; +import eu.wdaqua.qanary.commons.QanaryExceptionNoOrMultipleQuestions; import eu.wdaqua.qanary.commons.QanaryMessage; import eu.wdaqua.qanary.commons.QanaryQuestion; import eu.wdaqua.qanary.commons.QanaryUtils; +import eu.wdaqua.qanary.commons.triplestoreconnectors.QanaryTripleStoreConnector; import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse; import eu.wdaqua.qanary.communications.RestTemplateWithCaching; import eu.wdaqua.qanary.component.QanaryComponent; -import eu.wdaqua.qanary.component.QanaryComponentConfiguration; +import eu.wdaqua.qanary.exceptions.SparqlQueryFailed; /** * represents a wrapper of the DBpedia Spotlight service used as NED annotator @@ -60,15 +61,11 @@ public class DBpediaSpotlightNED extends QanaryComponent { RestTemplate restTemplate; - @Inject - private QanaryComponentConfiguration myQanaryComponentConfiguration; + private DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher; @Inject private DBpediaSpotlightConfiguration myDBpediaSpotlightConfiguration; - @Inject - private DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher; - private boolean ignoreSslCertificate; private final String applicationName; @@ -77,10 +74,12 @@ public class DBpediaSpotlightNED extends QanaryComponent { public DBpediaSpotlightNED( // @Value("${spring.application.name}") final String applicationName, // @Value("${dbpediaspotlight.endpoint.ssl.certificatevalidation.ignore:false}") final boolean ignore, // + @Autowired DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher, RestTemplateWithCaching restTemplate // ) { this.applicationName = applicationName; this.restTemplate = restTemplate; + this.myDBpediaSpotlightServiceFetcher = myDBpediaSpotlightServiceFetcher; this.setIgnoreSslCertificate(ignore); // check if files exists and are not empty @@ -155,12 +154,7 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception { myDBpediaSpotlightConfiguration.getConfidenceMinimum()); // STEP2: Call the DBpedia NED service - JsonArray resources = myDBpediaSpotlightServiceFetcher.getJsonFromService(myQuestion, // - myDBpediaSpotlightConfiguration.getEndpoint(), // - myDBpediaSpotlightConfiguration.getConfidenceMinimum(), // - restTemplate, // - myCacheOfResponses // - ); + JsonArray resources = myDBpediaSpotlightServiceFetcher.getJsonFromService(myQuestion); // get all found DBpedia resources List foundDBpediaResources = myDBpediaSpotlightServiceFetcher @@ -174,9 +168,15 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception { // TODO: create one larger SPARQL INSERT query that adds all discovered named // entities at once - for (FoundDBpediaResource found : foundDBpediaResources) { + String sparql = getSparqlInsertQuery(found, myQanaryQuestion); + myQanaryUtils.getQanaryTripleStoreConnector().update(sparql); + } + + return myQanaryMessage; + } + public String getSparqlInsertQuery(FoundDBpediaResource found, QanaryQuestion myQanaryQuestion) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException{ QuerySolutionMap bindingsForInsert = new QuerySolutionMap(); bindingsForInsert.add("graph", ResourceFactory.createResource(myQanaryQuestion.getOutGraph().toASCIIString())); @@ -194,10 +194,7 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception { // get the template of the INSERT query String sparql = this.loadQueryFromFile(FILENAME_INSERT_ANNOTATION, bindingsForInsert); logger.info("SPARQL query: {}", sparql); - myQanaryUtils.getQanaryTripleStoreConnector().update(sparql); - } - - return myQanaryMessage; + return sparql; } private String loadQueryFromFile(String filenameWithRelativePath, QuerySolutionMap bindings) throws IOException { diff --git a/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcher.java b/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcher.java index cd473bf7b..9db7b8237 100644 --- a/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcher.java +++ b/qanary-component-NED-DBpediaSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcher.java @@ -1,17 +1,5 @@ package eu.wdaqua.qanary.component.dbpediaspotlight.ned; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; -import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse; -import eu.wdaqua.qanary.component.dbpediaspotlight.ned.exceptions.DBpediaSpotlightJsonParsingNotPossible; -import net.minidev.json.JSONObject; -import org.apache.shiro.util.Assert; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpEntity; -import org.springframework.web.client.RestTemplate; - import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URI; @@ -21,14 +9,51 @@ import java.util.LinkedList; import java.util.List; +import org.apache.http.client.ClientProtocolException; +import org.apache.shiro.util.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; + +import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse; +import eu.wdaqua.qanary.communications.RestTemplateWithCaching; +import eu.wdaqua.qanary.component.dbpediaspotlight.ned.exceptions.DBpediaSpotlightJsonParsingNotPossible; +import net.minidev.json.JSONObject; + +@Component public class DBpediaSpotlightServiceFetcher { private static final Logger logger = LoggerFactory.getLogger(DBpediaSpotlightServiceFetcher.class); + private final String endpoint; + private final float minimumConfidence; + + private RestTemplateWithCaching restTemplate; + private CacheOfRestTemplateResponse myCacheOfResponses; + + public DBpediaSpotlightServiceFetcher( + @Autowired RestTemplateWithCaching myRestTemplate, + @Autowired CacheOfRestTemplateResponse myCacheOfResponses, + @Value("${dbpediaspotlight.endpoint}") String endpoint, + @Value("${dbpediaspotlight.confidence.minimum}") float minimumConfidence + ) { + + this.restTemplate = myRestTemplate; + this.myCacheOfResponses = myCacheOfResponses; + this.endpoint = endpoint; + this.minimumConfidence = minimumConfidence; + } + /** * fetch data from the configured DBpedia Spotlight endpoint * - * @param myQanaryQuestion - * @param myQanaryUtils * @param myQuestion * @param endpoint * @param minimumConfidence @@ -37,11 +62,10 @@ public class DBpediaSpotlightServiceFetcher { * @throws ClientProtocolException * @throws IOException */ - public JsonArray getJsonFromService(String myQuestion, String endpoint, float minimumConfidence, - RestTemplate restTemplate, CacheOfRestTemplateResponse myCacheOfResponses) throws Exception { + public JsonArray getJsonFromService(String myQuestion) throws Exception { - URI uri = createRequestUriWithParameters(myQuestion, endpoint, minimumConfidence); - HttpEntity response = fetchNamedEntitiesFromWebService(restTemplate, myCacheOfResponses, uri); + URI uri = createRequestUriWithParameters(myQuestion, this.endpoint, this.minimumConfidence); + HttpEntity response = fetchNamedEntitiesFromWebService(this.restTemplate, this.myCacheOfResponses, uri); JsonArray resources = getResourcesOfResponse(response, myQuestion); return resources; diff --git a/qanary-component-NED-DBpediaSpotlight/src/test/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcherTest.java b/qanary-component-NED-DBpediaSpotlight/src/test/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcherTest.java index 87dd6230b..042224acd 100644 --- a/qanary-component-NED-DBpediaSpotlight/src/test/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcherTest.java +++ b/qanary-component-NED-DBpediaSpotlight/src/test/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcherTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.List; @@ -14,9 +15,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; @@ -29,13 +32,19 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; +import eu.wdaqua.qanary.commons.QanaryExceptionNoOrMultipleQuestions; +import eu.wdaqua.qanary.commons.QanaryQuestion; import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse; import eu.wdaqua.qanary.communications.RestTemplateWithCaching; import eu.wdaqua.qanary.component.dbpediaspotlight.ned.exceptions.DBpediaSpotlightJsonParsingNotPossible; +import eu.wdaqua.qanary.exceptions.SparqlQueryFailed; import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; import net.minidev.json.parser.ParseException; +import static eu.wdaqua.qanary.commons.config.QanaryConfiguration.endpointKey; +import static org.mockito.ArgumentMatchers.any; + @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class) @WebAppConfiguration @@ -49,6 +58,13 @@ class DBpediaSpotlightServiceFetcherTest { @Autowired private CacheOfRestTemplateResponse myCacheOfResponse; + @Value("${spring.application.name}") + private String applicationName; + + private DBpediaSpotlightServiceFetcher mockedDBpediaSpotlightServiceFetcher; + private QanaryQuestion mockedQanaryQuestion; + private DBpediaSpotlightNED mockedDBpediaSpotlightNED; + static { // deactivate the live test of the real-world webservice System.setProperty("dbpediaspotlight.perform-live-check-on-component-start", "false"); @@ -56,8 +72,17 @@ class DBpediaSpotlightServiceFetcherTest { } @BeforeEach - public void init() throws URISyntaxException { + public void init() throws URISyntaxException, QanaryExceptionNoOrMultipleQuestions, SparqlQueryFailed, DBpediaSpotlightJsonParsingNotPossible, IOException { assert this.restTemplate != null : "restTemplate cannot be null"; + + this.mockedQanaryQuestion = Mockito.mock(QanaryQuestion.class); + Mockito.when(this.mockedQanaryQuestion.getOutGraph()).thenReturn(new URI(endpointKey)); + Mockito.when(this.mockedQanaryQuestion.getUri()).thenReturn(new URI("targetquestion")); + + this.mockedDBpediaSpotlightServiceFetcher = Mockito.mock(DBpediaSpotlightServiceFetcher.class, Mockito.RETURNS_DEEP_STUBS); + + this.mockedDBpediaSpotlightNED = Mockito.mock(DBpediaSpotlightNED.class); + Mockito.when(this.mockedDBpediaSpotlightNED.getSparqlInsertQuery(any(FoundDBpediaResource.class), any(QanaryQuestion.class))).thenCallRealMethod(); } /** @@ -86,7 +111,6 @@ void givenRestTemplate_whenRequested_thenLogAndModifyResponse() throws Interrupt callRestTemplateWithCaching(loginForm1, Cache.CACHED); // cache hit assertEquals(initialNumberOfRequests + 3, myCacheOfResponse.getNumberOfExecutedRequests()); - } /** @@ -159,12 +183,16 @@ public void setPassword(String password) { @Test void testParsingOfJsonResponseOffline() throws ParseException, DBpediaSpotlightJsonParsingNotPossible { - DBpediaSpotlightServiceFetcher myFetcher = new DBpediaSpotlightServiceFetcher(); + //DBpediaSpotlightServiceFetcher myFetcher = new DBpediaSpotlightServiceFetcher(); + Mockito.when(this.mockedDBpediaSpotlightServiceFetcher.getResourcesOfResponse(any(HttpEntity.class), any(String.class))).thenCallRealMethod(); + Mockito.when(this.mockedDBpediaSpotlightServiceFetcher.parseJsonBodyOfResponse(any(HttpEntity.class))).thenCallRealMethod(); JSONParser parser = new JSONParser(); JSONObject body = (JSONObject) parser.parse(knownValidResponseBody); HttpEntity response = new HttpEntity(body); - JsonArray resources = myFetcher.getResourcesOfResponse(response, knownValidResponseBody); + logger.info(response.toString()); + JsonArray resources = this.mockedDBpediaSpotlightServiceFetcher.getResourcesOfResponse(response, knownValidResponseBody); + logger.info(resources.toString()); assertEquals(4, resources.size()); @@ -175,23 +203,35 @@ void testParsingOfJsonResponseOffline() throws ParseException, DBpediaSpotlightJ } @Test - void testFoundResources() throws ParseException, DBpediaSpotlightJsonParsingNotPossible, URISyntaxException { - DBpediaSpotlightServiceFetcher myFetcher = new DBpediaSpotlightServiceFetcher(); - JSONParser parser = new JSONParser(); - - JSONObject body = (JSONObject) parser.parse(knownValidResponseBody); - HttpEntity response = new HttpEntity(body); - JsonArray resources = myFetcher.getResourcesOfResponse(response, knownValidResponseBody); - List foundDBpediaResources = myFetcher.getListOfResources(resources); - for (FoundDBpediaResource foundDBpediaResource : foundDBpediaResources) { - assertNotEquals(null, foundDBpediaResource); - assertTrue(foundDBpediaResource.getBegin() >= 0); - assertTrue(foundDBpediaResource.getEnd() >= 0); - assertTrue(foundDBpediaResource.getEnd() >= foundDBpediaResource.getBegin()); - assertTrue(foundDBpediaResource.getSimilarityScore() > 0); - assertTrue(foundDBpediaResource.getSupport() >= 0); - assertNotEquals(null, foundDBpediaResource.getResource()); - } + void testFoundResources() throws ParseException, DBpediaSpotlightJsonParsingNotPossible, URISyntaxException, QanaryExceptionNoOrMultipleQuestions, SparqlQueryFailed, IOException { + JSONParser parser = new JSONParser(); + JSONObject body = (JSONObject) parser.parse(knownValidResponseBody); + HttpEntity response = new HttpEntity(body); + JsonArray resources = this.mockedDBpediaSpotlightServiceFetcher.getResourcesOfResponse(response, knownValidResponseBody); + + List foundDBpediaResources = this.mockedDBpediaSpotlightServiceFetcher.getListOfResources(resources); + for (FoundDBpediaResource foundDBpediaResource : foundDBpediaResources) { + assertNotEquals(null, foundDBpediaResource); + assertTrue(foundDBpediaResource.getBegin() >= 0); + assertTrue(foundDBpediaResource.getEnd() >= 0); + assertTrue(foundDBpediaResource.getEnd() >= foundDBpediaResource.getBegin()); + assertTrue(foundDBpediaResource.getSimilarityScore() > 0); + assertTrue(foundDBpediaResource.getSupport() >= 0); + assertNotEquals(null, foundDBpediaResource.getResource()); + } + } + + @Test + void testGetSparqlInsertQuery() throws ParseException, DBpediaSpotlightJsonParsingNotPossible, URISyntaxException, QanaryExceptionNoOrMultipleQuestions, SparqlQueryFailed, IOException { + JSONParser parser = new JSONParser(); + JSONObject body = (JSONObject) parser.parse(knownValidResponseBody); + HttpEntity response = new HttpEntity(body); + JsonArray resources = this.mockedDBpediaSpotlightServiceFetcher.getResourcesOfResponse(response, knownValidResponseBody); + List foundDBpediaResources = this.mockedDBpediaSpotlightServiceFetcher.getListOfResources(resources); + for (FoundDBpediaResource foundDBpediaResource : foundDBpediaResources) { + String sparql = this.mockedDBpediaSpotlightNED.getSparqlInsertQuery(foundDBpediaResource, this.mockedQanaryQuestion); + assertNotNull(sparql); + assertNotEquals(0, sparql.length()); + } } - } diff --git a/qanary-component-NERD-SMAPH/pom.xml b/qanary-component-NERD-SMAPH/pom.xml index 6d726c8c8..c3ba2e76e 100644 --- a/qanary-component-NERD-SMAPH/pom.xml +++ b/qanary-component-NERD-SMAPH/pom.xml @@ -9,7 +9,7 @@ org.springframework.boot spring-boot-starter-parent - 2.6.7 + 2.7.0 17 @@ -83,22 +83,6 @@ spring-test test - - org.apache.jena - apache-jena-libs - pom - 3.0.0 - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - diff --git a/qanary-component-QBE-QAnswer/pom.xml b/qanary-component-QBE-QAnswer/pom.xml index a0c7d0017..b994b8f74 100644 --- a/qanary-component-QBE-QAnswer/pom.xml +++ b/qanary-component-QBE-QAnswer/pom.xml @@ -10,7 +10,7 @@ org.springframework.boot spring-boot-starter-parent - 2.6.7 + 2.7.0 11 @@ -86,22 +86,6 @@ com.google.code.gson gson - - org.apache.jena - apache-jena-libs - pom - 3.0.0 - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - org.springdoc diff --git a/qanary-component-QE-SparqlExecuter/pom.xml b/qanary-component-QE-SparqlExecuter/pom.xml index 26eb5b598..ec8c30693 100644 --- a/qanary-component-QE-SparqlExecuter/pom.xml +++ b/qanary-component-QE-SparqlExecuter/pom.xml @@ -5,7 +5,7 @@ 4.0.0 eu.wdaqua.qanary.component qanary-component-QE-SparqlExecuter - 3.1.7 + 3.2.0 org.springframework.boot spring-boot-starter-parent @@ -14,7 +14,6 @@ 17 [3.7.0,4.0.0) - 4.5.0 qanary qanary-component-qe-sparqlexecuter 1.4.13 @@ -88,23 +87,6 @@ json-path-assert test - - - - diff --git a/qanary-component-QE-SparqlExecuter/src/main/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuter.java b/qanary-component-QE-SparqlExecuter/src/main/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuter.java index 37fe63fb7..7ff9e950d 100644 --- a/qanary-component-QE-SparqlExecuter/src/main/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuter.java +++ b/qanary-component-QE-SparqlExecuter/src/main/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuter.java @@ -1,21 +1,31 @@ package eu.wdaqua.qanary.sparqlexecuter; -import eu.wdaqua.qanary.commons.QanaryMessage; -import eu.wdaqua.qanary.commons.QanaryQuestion; -import eu.wdaqua.qanary.commons.QanaryUtils; -import eu.wdaqua.qanary.commons.triplestoreconnectors.QanaryTripleStoreConnector; -import eu.wdaqua.qanary.component.QanaryComponent; -import org.apache.jena.query.*; +import static org.apache.jena.datatypes.xsd.XSDDatatype.XSDstring; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.jena.query.Query; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.QuerySolutionMap; +import org.apache.jena.query.ResultSet; +import org.apache.jena.query.ResultSetFormatter; import org.apache.jena.rdf.model.ResourceFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import java.io.ByteArrayOutputStream; -import java.net.URI; - -import static org.apache.jena.datatypes.xsd.XSDDatatype.XSDstring; +import eu.wdaqua.qanary.commons.QanaryExceptionNoOrMultipleQuestions; +import eu.wdaqua.qanary.commons.QanaryMessage; +import eu.wdaqua.qanary.commons.QanaryQuestion; +import eu.wdaqua.qanary.commons.QanaryUtils; +import eu.wdaqua.qanary.commons.triplestoreconnectors.QanaryTripleStoreConnector; +import eu.wdaqua.qanary.component.QanaryComponent; +import eu.wdaqua.qanary.exceptions.SparqlQueryFailed; @Component @@ -52,33 +62,35 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception { QanaryQuestion myQanaryQuestion = new QanaryQuestion<>(myQanaryMessage, myQanaryUtils.getQanaryTripleStoreConnector()); // STEP 1: get the required data from the triplestore - String myQuestion = myQanaryQuestion.getTextualRepresentation(); - URI myQuestionUri = myQanaryQuestion.getUri(); - - ResultSet resultset = myQanaryUtils.getQanaryTripleStoreConnector().select(QanaryTripleStoreConnector.getHighestScoreAnnotationOfAnswerInGraph(myQanaryMessage.getInGraph())); - String sparqlQuery = ""; - while (resultset.hasNext()) { - sparqlQuery = resultset.next().get("selectQueryThatShouldComputeTheAnswer").toString().replace("\\\"", "\"").replace("\\n", "\n"); - } - logger.info("Generated SPARQL query: {} ", sparqlQuery); - if(resultset.getRowNumber() > 1) { - logger.warn("There are {} SPARQL queries that might represent the correct answer. Just the first one is used.", resultset.getRowNumber()); - } + String sparqlQuery = getResultSparqlQuery(myQanaryUtils, myQanaryQuestion); // STEP 2: execute the first SPARQL query - String endpoint = ""; - if (sparqlQuery.contains("http://dbpedia.org")) { - endpoint = this.knowledgegraphEndpointDbpedia; - logger.info("use DBpedia endpoint"); - } else if (sparqlQuery.contains("http://www.wikidata.org")) { - endpoint = this.knowledgegraphEndpointWikidata; - logger.info("use Wikidata endpoint"); - } else { - logger.warn("knowledge graph was unknown"); + String endpoint = selectKnowledgeGraphEnpdoint(sparqlQuery); + if (endpoint == null) { return myQanaryMessage; } - // TODO: extend functionality to use qa:TargetDataset if present + String json = getQueryResultsAsJson(sparqlQuery, endpoint); + logger.info("Generated answers in RDF json: {}", json); + + // STEP 3: Push the the JSON object to the named graph reserved for the question + logger.info("Push the the JSON object to the named graph reserved for the answer"); + String sparqlInsertAnnotationOfAnswerJson = getSparqlInsertQuery(json, myQanaryQuestion); + logger.info("SPARQL insert for adding data to Qanary triplestore: {}", sparqlInsertAnnotationOfAnswerJson); + myQanaryUtils.getQanaryTripleStoreConnector().update(sparqlInsertAnnotationOfAnswerJson); + + return myQanaryMessage; + } + + /** + * @param sparqlQuery + * @param endpoint + * @return + * @throws UnsupportedEncodingException + * @throws SparqlQueryFailed + * @throws URISyntaxException + */ + public String getQueryResultsAsJson(String sparqlQuery, String endpoint) throws UnsupportedEncodingException, eu.wdaqua.qanary.sparqlexecuter.exception.SparqlQueryFailed, URISyntaxException{ Query query = QueryFactory.create(sparqlQuery); TripleStoreConnector myTripleStoreConnector = new TripleStoreConnector(new URI(endpoint)); String json; @@ -93,28 +105,54 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception { ResultSetFormatter.outputAsJSON(outputStream, result); } json = new String(outputStream.toByteArray(), "UTF-8"); + return json; + } + + public String selectKnowledgeGraphEnpdoint(String sparqlQuery){ + // TODO: extend functionality to use qa:TargetDataset if present + String endpoint = ""; + if (sparqlQuery.contains("http://dbpedia.org")) { + endpoint = this.knowledgegraphEndpointDbpedia; + logger.info("use DBpedia endpoint"); + } else if (sparqlQuery.contains("http://www.wikidata.org")) { + endpoint = this.knowledgegraphEndpointWikidata; + logger.info("use Wikidata endpoint"); + } else { + logger.warn("knowledge graph was unknown"); + return null; + } + return endpoint; + } - logger.info("Generated answers in RDF json: {}", json); - - // STEP 3: Push the the JSON object to the named graph reserved for the question - logger.info("Push the the JSON object to the named graph reserved for the answer"); + public String getResultSparqlQuery(QanaryUtils myQanaryUtils, QanaryQuestion myQanaryQuestion) throws SparqlQueryFailed, IOException{ + // TODO: this was changed (no longer using QanaryMessage.getInGraph()) + ResultSet resultset = myQanaryUtils.getQanaryTripleStoreConnector().select(QanaryTripleStoreConnector.getHighestScoreAnnotationOfAnswerInGraph(myQanaryQuestion.getOutGraph())); + String sparqlQuery = ""; + while (resultset.hasNext()) { + sparqlQuery = resultset.next().get("selectQueryThatShouldComputeTheAnswer").toString().replace("\\\"", "\"").replace("\\n", "\n"); + } + logger.info("Generated SPARQL query: {} ", sparqlQuery); + if(resultset.getRowNumber() > 1) { + logger.warn("There are {} SPARQL queries that might represent the correct answer. Just the first one is used.", resultset.getRowNumber()); + } + return sparqlQuery; + } + public String getSparqlInsertQuery(String json, QanaryQuestion myQanaryQuestion) throws IOException, QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed{ // define here the parameters for the SPARQL INSERT query QuerySolutionMap bindings = new QuerySolutionMap(); // use here the variable names defined in method insertAnnotationOfAnswerSPARQL bindings.add("graph", ResourceFactory.createResource(myQanaryQuestion.getOutGraph().toASCIIString())); bindings.add("targetQuestion", ResourceFactory.createResource(myQanaryQuestion.getUri().toASCIIString())); - bindings.add("jsonAnswer", ResourceFactory.createTypedLiteral(json.replace("\n", " ").replace("\"", "\\\""), XSDstring)); + // TODO: check string replacement => leads to failed parsing in other components! + //bindings.add("jsonAnswer", ResourceFactory.createTypedLiteral(json.replace("\n", "").replace("\"", "\\\""), XSDstring)); + bindings.add("jsonAnswer", ResourceFactory.createTypedLiteral(json.replace("\n", ""), XSDstring)); bindings.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName)); // get the template of the INSERT query String sparqlInsertAnnotationOfAnswerJson = QanaryTripleStoreConnector.insertAnnotationOfAnswerJson(bindings); - logger.info("SPARQL insert for adding data to Qanary triplestore: {}", sparqlInsertAnnotationOfAnswerJson); - - myQanaryUtils.getQanaryTripleStoreConnector().update(sparqlInsertAnnotationOfAnswerJson); - - return myQanaryMessage; + return sparqlInsertAnnotationOfAnswerJson; } } diff --git a/qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/QanaryServiceControllerTest.java b/qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/QanaryServiceControllerTest.java deleted file mode 100644 index 399c3b955..000000000 --- a/qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/QanaryServiceControllerTest.java +++ /dev/null @@ -1,80 +0,0 @@ -package eu.wdaqua.qanary.sparqlexecuter; - -import eu.wdaqua.qanary.commons.QanaryMessage; -import eu.wdaqua.qanary.commons.config.QanaryConfiguration; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit.jupiter.SpringExtension; -import org.springframework.test.context.web.WebAppConfiguration; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.web.context.WebApplicationContext; - -import java.net.URI; -import java.net.URISyntaxException; - -import static eu.wdaqua.qanary.commons.config.QanaryConfiguration.*; -import static org.junit.jupiter.api.Assertions.*; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -@ExtendWith(SpringExtension.class) -@SpringBootTest(classes = Application.class) -@WebAppConfiguration -class QanaryServiceControllerTest { - private static final Logger logger = LoggerFactory.getLogger(QanaryServiceControllerTest.class); - private MockMvc mockMvc; - @Autowired - private WebApplicationContext applicationContext; - - /** - * initialize local controller enabled for tests - */ - @BeforeEach - public void setUp() { - this.mockMvc = MockMvcBuilders.webAppContextSetup(this.applicationContext).build(); - } - - /** - * test description interface - * - * @throws Exception - */ - @Test - void testDescriptionAvailable() throws Exception { - mockMvc.perform(get(QanaryConfiguration.description)) // fetch - .andExpect(status().isOk()) // HTTP 200 - .andReturn(); // - } - - /** - * test correct message format - */ - @Test - void testMessageFromJson() { - // create message from json string - QanaryMessage message; - try { - message = new QanaryMessage(new URI(endpointKey), new URI(inGraphKey), new URI(outGraphKey)); - - URI endpointKeyUrlFromMessage = message.getEndpoint(); - assertNotNull(endpointKeyUrlFromMessage); - - URI endpointKeyUrlFromHere = new URI(endpointKey); - - // TODO: more tests to ensure mechanism - assertEquals(endpointKeyUrlFromMessage.toString(), endpointKeyUrlFromHere.toString()); - - } catch (URISyntaxException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - - } - -} \ No newline at end of file diff --git a/qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuterTest.java b/qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuterTest.java new file mode 100644 index 000000000..e33d82cdb --- /dev/null +++ b/qanary-component-QE-SparqlExecuter/src/test/java/eu/wdaqua/qanary/sparqlexecuter/SparqlExecuterTest.java @@ -0,0 +1,114 @@ +package eu.wdaqua.qanary.sparqlexecuter; + +import static eu.wdaqua.qanary.commons.config.QanaryConfiguration.endpointKey; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.jena.query.QuerySolution; +import org.apache.jena.query.ResultSet; +import org.apache.jena.rdf.model.RDFNode; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.web.WebAppConfiguration; + +import eu.wdaqua.qanary.commons.QanaryExceptionNoOrMultipleQuestions; +import eu.wdaqua.qanary.commons.QanaryQuestion; +import eu.wdaqua.qanary.commons.QanaryUtils; +import eu.wdaqua.qanary.commons.triplestoreconnectors.QanaryTripleStoreConnector; +import eu.wdaqua.qanary.exceptions.SparqlQueryFailed; + + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class) +@WebAppConfiguration +class SparqlExecuterTest { + + private static final Logger logger = LoggerFactory.getLogger(SparqlExecuterTest.class); + + @Value("${knowledgegraph.endpoint.wikidata}") + private String knowledgegraphEndpointWikidata; + + String knownValidResponseBody = "{ \"head\": { \"vars\": [ \"Who\" ] } , \"results\": { \"bindings\": [ { \"Who\": { \"type\": \"uri\" , \"value\": \"http://dbpedia.org/resource/Rembrandt\" } } ] } }"; + String knownValidSelectQuery = "SELECT DISTINCT ?o1 WHERE { ?o1 . } LIMIT 1000"; + + private SparqlExecuter mockedSparqlExecuter; + private QanaryQuestion mockedQanaryQuestion; + private QanaryUtils mockedQanaryUtils; + private QanaryTripleStoreConnector mockedQanaryTriplestoreConnector; + private RDFNode mockedRDFNode; + + @BeforeEach + public void init() throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException, eu.wdaqua.qanary.sparqlexecuter.exception.SparqlQueryFailed { + this.mockedSparqlExecuter = Mockito.mock(SparqlExecuter.class); + this.mockedQanaryQuestion = Mockito.mock(QanaryQuestion.class); + this.mockedQanaryUtils = Mockito.mock(QanaryUtils.class); + this.mockedQanaryTriplestoreConnector = Mockito.mock(QanaryTripleStoreConnector.class); + Mockito.when(this.mockedQanaryQuestion.getOutGraph()).thenReturn(new URI(endpointKey)); + Mockito.when(this.mockedQanaryQuestion.getUri()).thenReturn(new URI("targetquestion")); + Mockito.when(this.mockedSparqlExecuter.getSparqlInsertQuery(any(String.class), any(QanaryQuestion.class))).thenCallRealMethod(); + Mockito.when(this.mockedQanaryUtils.getQanaryTripleStoreConnector()).thenReturn(this.mockedQanaryTriplestoreConnector); + Mockito.when(this.mockedSparqlExecuter.getResultSparqlQuery(any(QanaryUtils.class), any(QanaryQuestion.class))).thenCallRealMethod(); + Mockito.when(this.mockedSparqlExecuter.getQueryResultsAsJson(any(String.class), any(String.class))).thenCallRealMethod(); + + ResultSet mockedResultSet = Mockito.mock(ResultSet.class); + QuerySolution mockedQuerySolution = Mockito.mock(QuerySolution.class); + + this.mockedRDFNode = Mockito.mock(RDFNode.class); + Mockito.when(mockedResultSet.hasNext()).thenReturn(true).thenReturn(false); // return only one result + Mockito.when(mockedResultSet.next()).thenReturn(mockedQuerySolution); Mockito.when(mockedQuerySolution.get(any(String.class))).thenReturn(mockedRDFNode); + + Mockito.when(this.mockedQanaryTriplestoreConnector.select(any(String.class))) + .thenReturn(mockedResultSet); + + } + + @Test + void testGetSparqlInsertQuery() throws IOException, QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed { + String json = knownValidResponseBody; + String sparql = this.mockedSparqlExecuter.getSparqlInsertQuery(json, this.mockedQanaryQuestion); + assertNotNull(sparql); + assertNotEquals(0, sparql.length()); + } + + @Test + void testGetResultSparqlQuery() throws SparqlQueryFailed, IOException { + // This test exists only to ensure that SparqlExecuter.getResultSparqlQuery() + // can handle the return type org.apache.jena.query.ResultSet. + // + // The actual query is not relevant! + + + Mockito.when(mockedRDFNode.toString()).thenReturn(this.knownValidSelectQuery); + + + String resultSparqlQuery = this.mockedSparqlExecuter.getResultSparqlQuery( + this.mockedQanaryUtils, this.mockedQanaryQuestion); + + assertNotNull(resultSparqlQuery); + assertNotEquals(0, resultSparqlQuery.length()); + } + + @Test + void testGetQueryResultsAsJson() throws eu.wdaqua.qanary.sparqlexecuter.exception.SparqlQueryFailed, UnsupportedEncodingException, URISyntaxException { + + String jsonString = this.mockedSparqlExecuter.getQueryResultsAsJson( + this.knownValidSelectQuery, this.knowledgegraphEndpointWikidata); + assertNotNull(jsonString); + assertNotEquals(0, jsonString.length()); + + } + +}