Skip to content

Commit

Permalink
Remove org.apache.jena from components (#296)
Browse files Browse the repository at this point in the history 
* refactor main process method to separate methods

* add checks for getSparqlInswertQuery() to tests

* add test for getSparqlInsertQuery

* remove org.apache.jena dependency from pom.xml

* remove org.apache.jena dependency from pom.xml

* add test for methods using org.apache.jena

* refactor process method
  • Loading branch information
@heinpa
heinpa committed Aug 10, 2023
1 parent cddba49 commit 87531c2
Show file tree
Hide file tree
Showing 11 changed files with 321 additions and 277 deletions.
30 changes: 1 addition & 29 deletions qanary-component-NED-DBpediaSpotlight/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>eu.wdaqua.qanary.component</groupId>
<artifactId>qanary-component-NED-DBpediaSpotlight</artifactId>
<version>3.3.1</version>
<version>3.4.0</version>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
Expand Down Expand Up @@ -86,34 +86,6 @@
<artifactId>spring-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.jena</groupId>
<artifactId>apache-jena-libs</artifactId>
<type>pom</type>
<version>3.0.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
Expand Down
17 changes: 3 additions & 14 deletions ...aSpotlight/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/Application.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
package eu.wdaqua.qanary.component.dbpediaspotlight.ned;

import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -16,8 +12,6 @@
import org.springframework.web.client.RestTemplate;

import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse;
import eu.wdaqua.qanary.communications.RestTemplateWithCaching;
import eu.wdaqua.qanary.component.QanaryComponent;
import eu.wdaqua.qanary.component.dbpediaspotlight.ned.exceptions.DBpediaSpotlightServiceNotAvailable;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.info.Info;
Expand Down Expand Up @@ -46,22 +40,18 @@ public static void main(String[] args) {

@Bean
public DBpediaSpotlightConfiguration myDBpediaSpotlightConfiguration( //
@Autowired DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher, //
@Value("${dbpediaspotlight.test-question}") String testQuestion, //
@Value("${dbpediaspotlight.confidence.minimum}") float confidenceMinimum, //
@Value("${dbpediaspotlight.endpoint:https://api.dbpedia-spotlight.org/en/annotate}") String endpoint, //
@Value("${dbpediaspotlight.perform-live-check-on-component-start:true}") boolean performLiveCheckOnComponentStart, //
@Value("${dbpediaspotlight.endpoint.ssl.certificatevalidation.ignore:false}") final boolean ignore
) throws DBpediaSpotlightServiceNotAvailable {
this.checkSpotlightServiceAvailability(testQuestion, endpoint, confidenceMinimum,
myDBpediaSpotlightServiceFetcher(), performLiveCheckOnComponentStart, ignore);
myDBpediaSpotlightServiceFetcher, performLiveCheckOnComponentStart, ignore);
return new DBpediaSpotlightConfiguration(confidenceMinimum, endpoint);
}

@Bean
public DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher() {
return new DBpediaSpotlightServiceFetcher();
}

private void checkSpotlightServiceAvailability(String testQuestion, String endpoint, float confidenceMinimum,
DBpediaSpotlightServiceFetcher dBpediaSpotlightServiceFetcher, boolean performLiveCheckOnComponentStart, boolean ignore)
throws DBpediaSpotlightServiceNotAvailable {
Expand All @@ -79,8 +69,7 @@ private void checkSpotlightServiceAvailability(String testQuestion, String endpo
restTemplate.setRequestFactory(DBpediaSpotlightNED.getRequestFactoryForSslVerficationDeactivation());
logger.warn("SSL certificate validation deactivated.");
}
dBpediaSpotlightServiceFetcher.getJsonFromService(testQuestion, endpoint, confidenceMinimum,
restTemplate, myCacheOfResponses);
dBpediaSpotlightServiceFetcher.getJsonFromService(testQuestion);
return;
} catch (Exception e) {
err = e.toString() + " " + e.getLocalizedMessage();
Expand Down
35 changes: 16 additions & 19 deletions ...ht/src/main/java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightNED.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package eu.wdaqua.qanary.component.dbpediaspotlight.ned;

import java.io.IOException;
import java.net.URISyntaxException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
Expand All @@ -10,8 +11,6 @@
import javax.inject.Inject;
import javax.net.ssl.SSLContext;

import eu.wdaqua.qanary.commons.triplestoreconnectors.QanaryTripleStoreConnector;

import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
Expand All @@ -29,13 +28,15 @@

import com.google.gson.JsonArray;

import eu.wdaqua.qanary.commons.QanaryExceptionNoOrMultipleQuestions;
import eu.wdaqua.qanary.commons.QanaryMessage;
import eu.wdaqua.qanary.commons.QanaryQuestion;
import eu.wdaqua.qanary.commons.QanaryUtils;
import eu.wdaqua.qanary.commons.triplestoreconnectors.QanaryTripleStoreConnector;
import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse;
import eu.wdaqua.qanary.communications.RestTemplateWithCaching;
import eu.wdaqua.qanary.component.QanaryComponent;
import eu.wdaqua.qanary.component.QanaryComponentConfiguration;
import eu.wdaqua.qanary.exceptions.SparqlQueryFailed;

/**
* represents a wrapper of the DBpedia Spotlight service used as NED annotator
Expand All @@ -60,15 +61,11 @@ public class DBpediaSpotlightNED extends QanaryComponent {

RestTemplate restTemplate;

@Inject
private QanaryComponentConfiguration myQanaryComponentConfiguration;
private DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher;

@Inject
private DBpediaSpotlightConfiguration myDBpediaSpotlightConfiguration;

@Inject
private DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher;

private boolean ignoreSslCertificate;

private final String applicationName;
Expand All @@ -77,10 +74,12 @@ public class DBpediaSpotlightNED extends QanaryComponent {
public DBpediaSpotlightNED( //
@Value("${spring.application.name}") final String applicationName, //
@Value("${dbpediaspotlight.endpoint.ssl.certificatevalidation.ignore:false}") final boolean ignore, //
@Autowired DBpediaSpotlightServiceFetcher myDBpediaSpotlightServiceFetcher,
RestTemplateWithCaching restTemplate //
) {
this.applicationName = applicationName;
this.restTemplate = restTemplate;
this.myDBpediaSpotlightServiceFetcher = myDBpediaSpotlightServiceFetcher;
this.setIgnoreSslCertificate(ignore);

// check if files exists and are not empty
Expand Down Expand Up @@ -155,12 +154,7 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception {
myDBpediaSpotlightConfiguration.getConfidenceMinimum());

// STEP2: Call the DBpedia NED service
JsonArray resources = myDBpediaSpotlightServiceFetcher.getJsonFromService(myQuestion, //
myDBpediaSpotlightConfiguration.getEndpoint(), //
myDBpediaSpotlightConfiguration.getConfidenceMinimum(), //
restTemplate, //
myCacheOfResponses //
);
JsonArray resources = myDBpediaSpotlightServiceFetcher.getJsonFromService(myQuestion);

// get all found DBpedia resources
List<FoundDBpediaResource> foundDBpediaResources = myDBpediaSpotlightServiceFetcher
Expand All @@ -174,9 +168,15 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception {

// TODO: create one larger SPARQL INSERT query that adds all discovered named
// entities at once

for (FoundDBpediaResource found : foundDBpediaResources) {
String sparql = getSparqlInsertQuery(found, myQanaryQuestion);
myQanaryUtils.getQanaryTripleStoreConnector().update(sparql);
}

return myQanaryMessage;
}

public String getSparqlInsertQuery(FoundDBpediaResource found, QanaryQuestion<String> myQanaryQuestion) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException{
QuerySolutionMap bindingsForInsert = new QuerySolutionMap();
bindingsForInsert.add("graph",
ResourceFactory.createResource(myQanaryQuestion.getOutGraph().toASCIIString()));
Expand All @@ -194,10 +194,7 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception {
// get the template of the INSERT query
String sparql = this.loadQueryFromFile(FILENAME_INSERT_ANNOTATION, bindingsForInsert);
logger.info("SPARQL query: {}", sparql);
myQanaryUtils.getQanaryTripleStoreConnector().update(sparql);
}

return myQanaryMessage;
return sparql;
}

private String loadQueryFromFile(String filenameWithRelativePath, QuerySolutionMap bindings) throws IOException {
Expand Down
60 changes: 42 additions & 18 deletions .../java/eu/wdaqua/qanary/component/dbpediaspotlight/ned/DBpediaSpotlightServiceFetcher.java
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,5 @@
package eu.wdaqua.qanary.component.dbpediaspotlight.ned;

import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;
import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse;
import eu.wdaqua.qanary.component.dbpediaspotlight.ned.exceptions.DBpediaSpotlightJsonParsingNotPossible;
import net.minidev.json.JSONObject;
import org.apache.shiro.util.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.web.client.RestTemplate;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
Expand All @@ -21,14 +9,51 @@
import java.util.LinkedList;
import java.util.List;

import org.apache.http.client.ClientProtocolException;
import org.apache.shiro.util.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;

import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;

import eu.wdaqua.qanary.communications.CacheOfRestTemplateResponse;
import eu.wdaqua.qanary.communications.RestTemplateWithCaching;
import eu.wdaqua.qanary.component.dbpediaspotlight.ned.exceptions.DBpediaSpotlightJsonParsingNotPossible;
import net.minidev.json.JSONObject;

@Component
public class DBpediaSpotlightServiceFetcher {
private static final Logger logger = LoggerFactory.getLogger(DBpediaSpotlightServiceFetcher.class);

private final String endpoint;
private final float minimumConfidence;

private RestTemplateWithCaching restTemplate;
private CacheOfRestTemplateResponse myCacheOfResponses;

public DBpediaSpotlightServiceFetcher(
@Autowired RestTemplateWithCaching myRestTemplate,
@Autowired CacheOfRestTemplateResponse myCacheOfResponses,
@Value("${dbpediaspotlight.endpoint}") String endpoint,
@Value("${dbpediaspotlight.confidence.minimum}") float minimumConfidence
) {

this.restTemplate = myRestTemplate;
this.myCacheOfResponses = myCacheOfResponses;
this.endpoint = endpoint;
this.minimumConfidence = minimumConfidence;
}

/**
* fetch data from the configured DBpedia Spotlight endpoint
*
* @param myQanaryQuestion
* @param myQanaryUtils
* @param myQuestion
* @param endpoint
* @param minimumConfidence
Expand All @@ -37,11 +62,10 @@ public class DBpediaSpotlightServiceFetcher {
* @throws ClientProtocolException
* @throws IOException
*/
public JsonArray getJsonFromService(String myQuestion, String endpoint, float minimumConfidence,
RestTemplate restTemplate, CacheOfRestTemplateResponse myCacheOfResponses) throws Exception {
public JsonArray getJsonFromService(String myQuestion) throws Exception {

URI uri = createRequestUriWithParameters(myQuestion, endpoint, minimumConfidence);
HttpEntity<JSONObject> response = fetchNamedEntitiesFromWebService(restTemplate, myCacheOfResponses, uri);
URI uri = createRequestUriWithParameters(myQuestion, this.endpoint, this.minimumConfidence);
HttpEntity<JSONObject> response = fetchNamedEntitiesFromWebService(this.restTemplate, this.myCacheOfResponses, uri);
JsonArray resources = getResourcesOfResponse(response, myQuestion);

return resources;
Expand Down
Loading

0 comments on commit 87531c2

Please sign in to comment.