Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate OAIPMH server to Elasticsearch / Spring MVC #7583

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
136 changes: 101 additions & 35 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/Lib.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand All @@ -23,34 +23,46 @@

package org.fao.geonet.kernel.oaipmh;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jeeves.constants.Jeeves;
import jeeves.server.ServiceConfig;
import jeeves.server.context.ServiceContext;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.SearchHit;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.schema.iso19139.ISO19139SchemaPlugin;
import org.fao.geonet.utils.Xml;
import org.fao.oaipmh.exceptions.OaiPmhException;
import org.jdom.Element;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

import static org.fao.geonet.kernel.search.EsSearchManager.FIELDLIST_CORE;

//=============================================================================

public class Lib {
public static final String SESSION_OBJECT = "oai-list-records-result";

private Lib() {

}
//---------------------------------------------------------------------------
//---
//--- API methods
//---
//---------------------------------------------------------------------------
private static ServiceConfig dummyConfig = new ServiceConfig();

//--------------------------------------------------------------------------

public static boolean existsConverter(Path schemaDir, String prefix) {
Path f = schemaDir.resolve("convert").resolve(prefix + ".xsl");

Check failure

Code scanning / CodeQL

Uncontrolled data used in path expression High

This path depends on a
user-provided value
.
return Files.exists(f);
}

Expand Down Expand Up @@ -83,7 +95,7 @@

//--- do an XSL transformation

Path styleSheet = schemaDir.resolve("convert").resolve(targetFormat);

Check failure

Code scanning / CodeQL

Uncontrolled data used in path expression High

This path depends on a
user-provided value
.

return Xml.transform(root, styleSheet);
}
Expand All @@ -91,38 +103,29 @@
//---------------------------------------------------------------------------

public static List<Integer> search(ServiceContext context, Element params) throws Exception {
throw new UnsupportedOperationException("ES search does not support OAI yet");

// GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
// SearchManager sm = gc.getBean(SearchManager.class);
//
// try (MetaSearcher searcher = sm.newSearcher(SearcherType.LUCENE, Geonet.File.SEARCH_LUCENE)) {
//
// if (context.isDebugEnabled())
// context.debug("Searching with params:\n" + Xml.getString(params));
//
// searcher.search(context, params, dummyConfig);
//
// params.addContent(new Element("fast").setText("true"));
// params.addContent(new Element("from").setText("1"));
// params.addContent(new Element("to").setText(searcher.getSize() + ""));
//
// context.info("Records found : " + searcher.getSize());
//
// Element records = searcher.present(context, params, dummyConfig);
//
// records.getChild("summary").detach();
//
// List<Integer> result = new ArrayList<Integer>();
//
// for (Object o : records.getChildren()) {
// Element rec = (Element) o;
// Element info = rec.getChild("info", Edit.NAMESPACE);
//
// result.add(Integer.parseInt(info.getChildText("id")));
// }
// return result;
// }
EsSearchManager searchMan = context.getBean(EsSearchManager.class);

JsonNode esJsonQuery = createSearchQuery(params);

// Get results count
SearchResponse queryResult = searchMan.query(
esJsonQuery,
FIELDLIST_CORE,
0, 1);

long total = queryResult.getHits().getTotalHits().value;

queryResult = searchMan.query(
esJsonQuery,
FIELDLIST_CORE,
0, (int) total);

List<Integer> result = new ArrayList<>();

for (SearchHit hit : queryResult.getHits()) {
result.add(Integer.parseInt(hit.getSourceAsMap().get(Geonet.IndexFieldNames.ID).toString()));
}
return result;
}

//---------------------------------------------------------------------------
Expand Down Expand Up @@ -152,6 +155,69 @@

return error;
}

static JsonNode createSearchQuery(Element params) throws JsonProcessingException {
final String PARAM_SET = "category";
final String PARAM_FROM = "extFrom";
final String PARAM_UNTIL = "extTo";
final String PARAM_METADATAPREFIX = "_schema";

String jsonQuery = "{" +
" \"bool\": {" +
" \"must\": [" +
" {" +
" \"terms\": {" +
" \"isTemplate\": [\"n\"]" +
" }" +
" }, " +
" {" +
" \"term\": {" +
" \"documentStandard\": {" +
" \"value\": \"%s\"" +
" }" +
" }" +
" }%s%s" +
" ]" +
" }" +
"}";

String categoryQuery = "";
if (params.getChild(PARAM_SET) != null) {
categoryQuery = String.format(" ,{" +
" \"term\": {" +
" \"cat\": {" +
" \"value\": \"%s\"" +
" }" +
" }" +
" }", params.getChild(PARAM_SET).getValue());
}

String temporalExtentQuery = "";
if ((params.getChild(PARAM_FROM) != null) ||
(params.getChild("extTo") != null)) {

String fromQuery = "";
if (params.getChild(PARAM_FROM) != null) {
fromQuery = String.format(" \"gte\": \"%s\",", params.getChild(PARAM_FROM).getValue());
}

String untilQuery = "";
if (params.getChild(PARAM_UNTIL) != null) {
untilQuery = String.format(" \"lte\": \"%s\",", params.getChild(PARAM_UNTIL).getValue()) ;
}

temporalExtentQuery = String.format(" ,{\"range\": {\"resourceTemporalDateRange\": {" +
" %s" +
" %s" +
" \"relation\": \"intersects\"" +
" }}}", fromQuery, untilQuery);
}

ObjectMapper objectMapper = new ObjectMapper();

String schema = (params.getChild(PARAM_METADATAPREFIX) != null) ? params.getChild(PARAM_METADATAPREFIX).getValue() : ISO19139SchemaPlugin.IDENTIFIER;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fallback seems useless to me since the request will give an error if no metadataPrefix parameter is given

return objectMapper.readTree(String.format(jsonQuery, schema, categoryQuery, temporalExtentQuery));
}
}

//=============================================================================
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -65,7 +65,7 @@ public class OaiPmhDispatcher {
//--- Constructor
//---
//---------------------------------------------------------------------------
private HashMap<String, OaiPmhService> hmServices = new HashMap<String, OaiPmhService>();
private HashMap<String, OaiPmhService> hmServices = new HashMap<>();

//---------------------------------------------------------------------------

Expand Down Expand Up @@ -96,25 +96,25 @@ private void register(OaiPmhService s) {
//---
//---------------------------------------------------------------------------

public Element dispatch(Element request, ServiceContext context) {
Element response = dispatchI(request, context);
public Element dispatch(OaiPmhParams oaiPmhParams, ServiceContext context) {
Element response = dispatchI(oaiPmhParams, context);
validateResponse(context, response);

return response;
}

//---------------------------------------------------------------------------

private Element dispatchI(Element request, ServiceContext context) {
private Element dispatchI(OaiPmhParams oaiPmhParams, ServiceContext context) {
String url = null;

Map<String, String> params = null;

SettingInfo si = context.getBean(SettingInfo.class);

try {
url = si.getSiteUrl() + context.getBaseUrl() + "/" + Jeeves.Prefix.SERVICE + "/en/" + context.getService();
params = OaiPmhFactory.extractParams(request);
url = si.getSiteUrl() + context.getBaseUrl() + "/" + Jeeves.Prefix.SERVICE + "/" + context.getService();
params = oaiPmhParams.asMap();

AbstractRequest req = OaiPmhFactory.parse(context.getApplicationContext(), params);
OaiPmhService srv = hmServices.get(req.getVerb());
Expand Down
132 changes: 132 additions & 0 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/OaiPmhParams.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
* Copyright (C) 2001-2023 Food and Agriculture Organization of the
* United Nations (FAO-UN), United Nations World Food Programme (WFP)
* and United Nations Environment Programme (UNEP)
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or (at
* your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
*
* Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
* Rome - Italy. email: [email protected]
*/

package org.fao.geonet.kernel.oaipmh;

import org.apache.commons.lang.StringUtils;
import org.fao.oaipmh.OaiPmh;

import java.util.HashMap;
import java.util.Map;

public class OaiPmhParams {
private String verb;
private String metadataPrefix;
private String identifier;
private String set;
private String from;
private String until;
private String resumptionToken;

public String getVerb() {
return verb;
}

public void setVerb(String verb) {
this.verb = verb;
}

public String getMetadataPrefix() {
return metadataPrefix;
}

public void setMetadataPrefix(String metadataPrefix) {
this.metadataPrefix = metadataPrefix;
}

public String getIdentifier() {
return identifier;
}

public void setIdentifier(String identifier) {
this.identifier = identifier;
}

public String getSet() {
return set;
}

public void setSet(String set) {
this.set = set;
}

public String getFrom() {
return from;
}

public void setFrom(String from) {
this.from = from;
}

public String getUntil() {
return until;
}

public void setUntil(String until) {
this.until = until;
}

public String getResumptionToken() {
return resumptionToken;
}

public void setResumptionToken(String resumptionToken) {
this.resumptionToken = resumptionToken;
}

public Map<String, String> asMap() {
Map<String, String> params = new HashMap<>();

if (StringUtils.isNotEmpty(verb)) {
params.put(OaiPmh.ParamNames.VERB, verb);
}

if (StringUtils.isNotEmpty(metadataPrefix)) {
//--- remove any non letter, number, dot or dash character
String processedMetadataPrefix = metadataPrefix.replaceAll("[^a-zA-Z0-9.-]", "");
params.put(OaiPmh.ParamNames.METADATA_PREFIX, processedMetadataPrefix);
}

if (StringUtils.isNotEmpty(identifier)) {
params.put(OaiPmh.ParamNames.IDENTIFIER, identifier);
}

if (StringUtils.isNotEmpty(set)) {
params.put(OaiPmh.ParamNames.SET, set);
}

if (StringUtils.isNotEmpty(from)) {
params.put(OaiPmh.ParamNames.FROM, from);
}

if (StringUtils.isNotEmpty(until)) {
params.put(OaiPmh.ParamNames.UNTIL, until);
}

if (StringUtils.isNotEmpty(resumptionToken)) {
params.put(OaiPmh.ParamNames.RESUMPTION_TOKEN, resumptionToken);
}

return params;
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -31,12 +31,12 @@
//=============================================================================

public interface OaiPmhService {
public String getVerb();
String getVerb();

/**
* Executes the service on given input request
*/
public AbstractResponse execute(AbstractRequest request, ServiceContext context)
AbstractResponse execute(AbstractRequest request, ServiceContext context)
throws Exception;
}

Expand Down
Loading
Loading