Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Migrate OAIPMH server to Elasticsearch / Spring MVC #7583

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Migrate OAIPMH server to Elasticsearch / Spring MVC #7583

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
16b562d
Migrate OAIPMH server to Elasticsearch
josegar74 Dec 28, 2023
c0849bd
Migrate OAIPMH server to Elasticsearch / Update documentation
josegar74 Dec 28, 2023
65a7384
Migrate OAIPMH server API to Spring MVC
josegar74 Dec 28, 2023
07cd230
Migrate OAIPMH server to Elasticsearch / fix total calculation
josegar74 Dec 28, 2023
4c4943e
OAIPMH server / Sonarlint improvements
josegar74 Dec 28, 2023
17a700d
OAIPMH server / Fix base URL
josegar74 Jan 9, 2024
726ba00
OAIPMH server / Model API parameters and Sonarlint improvements
josegar74 Jan 9, 2024
727e328
OAIPMH server / Remove non valid chars from the value to match the xs…
josegar74 Jan 10, 2024
3f291bd
Merge remote-tracking branch 'upstream/main' into oaipmh-server
josegar74 Feb 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 101 additions & 35 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/Lib.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand All @@ -23,29 +23,41 @@

package org.fao.geonet.kernel.oaipmh;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jeeves.constants.Jeeves;
import jeeves.server.ServiceConfig;
import jeeves.server.context.ServiceContext;

import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.SearchHit;
import org.fao.geonet.constants.Geonet;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.schema.iso19139.ISO19139SchemaPlugin;
import org.fao.geonet.utils.Xml;
import org.fao.oaipmh.exceptions.OaiPmhException;
import org.jdom.Element;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

import static org.fao.geonet.kernel.search.EsSearchManager.FIELDLIST_CORE;

//=============================================================================

public class Lib {
public static final String SESSION_OBJECT = "oai-list-records-result";

private Lib() {

}
//---------------------------------------------------------------------------
//---
//--- API methods
//---
//---------------------------------------------------------------------------
private static ServiceConfig dummyConfig = new ServiceConfig();

//--------------------------------------------------------------------------

Expand Down Expand Up @@ -91,38 +103,29 @@ public static Element transform(Path schemaDir, Element env, Element md, String
//---------------------------------------------------------------------------

public static List<Integer> search(ServiceContext context, Element params) throws Exception {
throw new UnsupportedOperationException("ES search does not support OAI yet");

// GeonetContext gc = (GeonetContext) context.getHandlerContext(Geonet.CONTEXT_NAME);
// SearchManager sm = gc.getBean(SearchManager.class);
//
// try (MetaSearcher searcher = sm.newSearcher(SearcherType.LUCENE, Geonet.File.SEARCH_LUCENE)) {
//
// if (context.isDebugEnabled())
// context.debug("Searching with params:\n" + Xml.getString(params));
//
// searcher.search(context, params, dummyConfig);
//
// params.addContent(new Element("fast").setText("true"));
// params.addContent(new Element("from").setText("1"));
// params.addContent(new Element("to").setText(searcher.getSize() + ""));
//
// context.info("Records found : " + searcher.getSize());
//
// Element records = searcher.present(context, params, dummyConfig);
//
// records.getChild("summary").detach();
//
// List<Integer> result = new ArrayList<Integer>();
//
// for (Object o : records.getChildren()) {
// Element rec = (Element) o;
// Element info = rec.getChild("info", Edit.NAMESPACE);
//
// result.add(Integer.parseInt(info.getChildText("id")));
// }
// return result;
// }
EsSearchManager searchMan = context.getBean(EsSearchManager.class);

JsonNode esJsonQuery = createSearchQuery(params);

// Get results count
SearchResponse queryResult = searchMan.query(
esJsonQuery,
FIELDLIST_CORE,
0, 1);

long total = queryResult.getHits().getTotalHits().value;

queryResult = searchMan.query(
esJsonQuery,
FIELDLIST_CORE,
0, (int) total);

List<Integer> result = new ArrayList<>();

for (SearchHit hit : queryResult.getHits()) {
result.add(Integer.parseInt(hit.getSourceAsMap().get(Geonet.IndexFieldNames.ID).toString()));
}
return result;
}

//---------------------------------------------------------------------------
Expand Down Expand Up @@ -152,6 +155,69 @@ public static Element toJeevesException(OaiPmhException e) {

return error;
}

static JsonNode createSearchQuery(Element params) throws JsonProcessingException {
final String PARAM_SET = "category";
final String PARAM_FROM = "extFrom";
final String PARAM_UNTIL = "extTo";
final String PARAM_METADATAPREFIX = "_schema";

String jsonQuery = "{" +
" \"bool\": {" +
" \"must\": [" +
" {" +
" \"terms\": {" +
" \"isTemplate\": [\"n\"]" +
" }" +
" }, " +
" {" +
" \"term\": {" +
" \"documentStandard\": {" +
" \"value\": \"%s\"" +
" }" +
" }" +
" }%s%s" +
" ]" +
" }" +
"}";

String categoryQuery = "";
if (params.getChild(PARAM_SET) != null) {
categoryQuery = String.format(" ,{" +
" \"term\": {" +
" \"cat\": {" +
" \"value\": \"%s\"" +
" }" +
" }" +
" }", params.getChild(PARAM_SET).getValue());
}

String temporalExtentQuery = "";
if ((params.getChild(PARAM_FROM) != null) ||
(params.getChild("extTo") != null)) {

String fromQuery = "";
if (params.getChild(PARAM_FROM) != null) {
fromQuery = String.format(" \"gte\": \"%s\",", params.getChild(PARAM_FROM).getValue());
}

String untilQuery = "";
if (params.getChild(PARAM_UNTIL) != null) {
untilQuery = String.format(" \"lte\": \"%s\",", params.getChild(PARAM_UNTIL).getValue()) ;
}

temporalExtentQuery = String.format(" ,{\"range\": {\"resourceTemporalDateRange\": {" +
" %s" +
" %s" +
" \"relation\": \"intersects\"" +
" }}}", fromQuery, untilQuery);
}

ObjectMapper objectMapper = new ObjectMapper();

String schema = (params.getChild(PARAM_METADATAPREFIX) != null) ? params.getChild(PARAM_METADATAPREFIX).getValue() : ISO19139SchemaPlugin.IDENTIFIER;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fallback seems useless to me since the request will give an error if no metadataPrefix parameter is given

return objectMapper.readTree(String.format(jsonQuery, schema, categoryQuery, temporalExtentQuery));
}
}

//=============================================================================
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/OaiPmhDispatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -65,7 +65,7 @@ public class OaiPmhDispatcher {
//--- Constructor
//---
//---------------------------------------------------------------------------
private HashMap<String, OaiPmhService> hmServices = new HashMap<String, OaiPmhService>();
private HashMap<String, OaiPmhService> hmServices = new HashMap<>();

//---------------------------------------------------------------------------

Expand Down
6 changes: 3 additions & 3 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/OaiPmhService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -31,12 +31,12 @@
//=============================================================================

public interface OaiPmhService {
public String getVerb();
String getVerb();

/**
* Executes the service on given input request
*/
public AbstractResponse execute(AbstractRequest request, ServiceContext context)
AbstractResponse execute(AbstractRequest request, ServiceContext context)
throws Exception;
}

Expand Down
17 changes: 8 additions & 9 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/ResumptionTokenCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2001-2016 Food and Agriculture Organization of the
* Copyright (C) 2001-2023 Food and Agriculture Organization of the
* United Nations (FAO-UN), United Nations World Food Programme (WFP)
* and United Nations Environment Programme (UNEP)
*
Expand Down Expand Up @@ -36,9 +36,7 @@
import java.util.TimeZone;

public class ResumptionTokenCache extends Thread {


public final static int CACHE_EXPUNGE_DELAY = 10 * 1000; // 10 seconds
public static final int CACHE_EXPUNGE_DELAY = 10 * 1000; // 10 seconds

private Map<String, GeonetworkResumptionToken> map;
private static Object stopper = new Object();
Expand Down Expand Up @@ -93,6 +91,7 @@ public int getCachemaxsize() {
return settingMan.getValueAsInt("system/oai/cachesize");
}

@Override
public void run() {
synchronized (stopper) {
while (running && !isInterrupted()) {
Expand All @@ -113,8 +112,8 @@ private synchronized void expunge() {

Date now = getUTCTime();

for (Map.Entry entry : map.entrySet()) {
if (((GeonetworkResumptionToken) entry.getValue()).getExpirDate().toDate().getTime() / 1000 < (now.getTime() / 1000)) {
for (Map.Entry<String, GeonetworkResumptionToken> entry : map.entrySet()) {
if (entry.getValue().getExpirDate().toDate().getTime() / 1000 < (now.getTime() / 1000)) {
map.remove(entry.getKey());
if (Log.isDebugEnabled(Geonet.OAI_HARVESTER))
Log.debug(Geonet.OAI_HARVESTER, "OAI cache ::expunge removing:" + entry.getKey());
Expand All @@ -131,11 +130,11 @@ private void removeLast() {
long oldest = Long.MAX_VALUE;
Object oldkey = "";

for (Map.Entry entry : map.entrySet()) {
for (Map.Entry<String, GeonetworkResumptionToken> entry : map.entrySet()) {

if (((GeonetworkResumptionToken) entry.getValue()).getExpirDate().getSeconds() < oldest) {
if (entry.getValue().getExpirDate().getSeconds() < oldest) {
oldkey = entry.getKey();
oldest = ((GeonetworkResumptionToken) entry.getValue()).getExpirDate().getSeconds();
oldest = entry.getValue().getExpirDate().getSeconds();
}
}

Expand Down
16 changes: 6 additions & 10 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/services/AbstractTokenLister.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2001-2016 Food and Agriculture Organization of the
* Copyright (C) 2001-2023 Food and Agriculture Organization of the
* United Nations (FAO-UN), United Nations World Food Programme (WFP)
* and United Nations Environment Programme (UNEP)
*
Expand Down Expand Up @@ -57,7 +57,7 @@ public abstract class AbstractTokenLister implements OaiPmhService {
private SettingManager settingMan;
private SchemaManager schemaMan;

public AbstractTokenLister(ResumptionTokenCache cache, SettingManager sm, SchemaManager scm) {
protected AbstractTokenLister(ResumptionTokenCache cache, SettingManager sm, SchemaManager scm) {
this.cache = cache;
this.settingMan = sm;
this.schemaMan = scm;
Expand Down Expand Up @@ -114,10 +114,8 @@ public AbstractResponse execute(AbstractRequest request,

TokenListRequest req = (TokenListRequest) request;

//UserSession session = context.getUserSession();
SearchResult result;

//String token = req.getResumptionToken();
String strToken = req.getResumptionToken();
GeonetworkResumptionToken token = null;

Expand Down Expand Up @@ -165,11 +163,12 @@ public AbstractResponse execute(AbstractRequest request,
params.addContent(new Element("_schema").setText(schema));
result.addIds(Lib.search(context, (Element) params.clone()));
}
if (schemas.size() == 0) result.setIds(new ArrayList<Integer>());
if (schemas.isEmpty()) result.setIds(new ArrayList<>());
}

if (result.getIds().size() == 0)
if (result.getIds().isEmpty()) {
throw new NoRecordsMatchException("No results");
}

// we only need a new token if the result set is big enough
if (result.getIds().size() > getMaxRecords()) {
Expand All @@ -178,7 +177,6 @@ public AbstractResponse execute(AbstractRequest request,
}

} else {
//result = (SearchResult) session.getProperty(Lib.SESSION_OBJECT);
token = cache.getResumptionToken(GeonetworkResumptionToken.buildKey(req));
if (Log.isDebugEnabled(Geonet.OAI_HARVESTER))
Log.debug(Geonet.OAI_HARVESTER, "OAI ListRecords : using ResumptionToken :" + GeonetworkResumptionToken.buildKey(req));
Expand All @@ -188,7 +186,6 @@ public AbstractResponse execute(AbstractRequest request,

result = token.getRes();

//pos = result.parseToken(token);
pos = GeonetworkResumptionToken.getPos(req);
}

Expand All @@ -198,7 +195,6 @@ public AbstractResponse execute(AbstractRequest request,
if (token == null && res.getSize() == 0)
throw new NoRecordsMatchException("No results");

//result.setupToken(res, pos);
if (token != null) token.setupToken(pos);
res.setResumptionToken(token);

Expand All @@ -214,7 +210,7 @@ public AbstractResponse execute(AbstractRequest request,
*/

private List<String> getSchemasThatCanConvertTo(String prefix) {
List<String> result = new ArrayList<String>();
List<String> result = new ArrayList<>();
for (String schema : schemaMan.getSchemas()) {
if (Lib.existsConverter(schemaMan.getSchemaDir(schema), prefix)) {
result.add(schema);
Expand Down
17 changes: 8 additions & 9 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/services/GetRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -79,14 +79,13 @@ public static Record buildRecordStat(ServiceContext context, Specification<Metad

if (prefix.equals(schema)) {
Attribute schemaLocAtt = sm.getSchemaLocation(schema, context);
if (schemaLocAtt != null) {
if (md.getAttribute(schemaLocAtt.getName(), schemaLocAtt.getNamespace()) == null) {
md.setAttribute(schemaLocAtt);
// make sure namespace declaration for schemalocation is present -
// remove it first (does nothing if not there) then add it
md.removeNamespaceDeclaration(schemaLocAtt.getNamespace());
md.addNamespaceDeclaration(schemaLocAtt.getNamespace());
}
if (schemaLocAtt != null && (md.getAttribute(schemaLocAtt.getName(), schemaLocAtt.getNamespace()) == null)) {
md.setAttribute(schemaLocAtt);
// make sure namespace declaration for schemalocation is present -
// remove it first (does nothing if not there) then add it
md.removeNamespaceDeclaration(schemaLocAtt.getNamespace());
md.addNamespaceDeclaration(schemaLocAtt.getNamespace());

}
} else {
Path schemaDir = sm.getSchemaDir(schema);
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/java/org/fao/geonet/kernel/oaipmh/services/Identify.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//=============================================================================
//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the
//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the
//=== United Nations (FAO-UN), United Nations World Food Programme (WFP)
//=== and United Nations Environment Programme (UNEP)
//===
Expand Down Expand Up @@ -69,7 +69,7 @@ public AbstractResponse execute(AbstractRequest request, ServiceContext context)

//---------------------------------------------------------------------------

private ISODate getEarliestDS(ServiceContext context) throws Exception {
private ISODate getEarliestDS(ServiceContext context) {
final AbstractMetadata oldestByChangeDate = context.getBean(MetadataRepository.class).findOldestByChangeDate();

//--- if we don't have metadata, just return 'now'
Expand Down
Loading
Loading