diff --git a/core/pom.xml b/core/pom.xml index 87ac4c15b3b..6e20fb6bc3f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -498,6 +498,11 @@ com.fasterxml.jackson.datatype jackson-datatype-hibernate4 + + org.geonetwork-opensource + es-core + 3.3.0-SNAPSHOT + diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java index b5ce56c8f1c..d9353bbc282 100644 --- a/core/src/main/java/org/fao/geonet/kernel/DataManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/DataManager.java @@ -88,6 +88,7 @@ import org.fao.geonet.exceptions.ServiceNotAllowedEx; import org.fao.geonet.exceptions.XSDValidationErrorEx; import org.fao.geonet.kernel.schema.MetadataSchema; +import org.fao.geonet.kernel.search.ISearchManager; import org.fao.geonet.kernel.search.SearchManager; import org.fao.geonet.kernel.search.index.IndexingList; import org.fao.geonet.kernel.setting.SettingManager; @@ -415,7 +416,7 @@ public synchronized void init(ServiceContext context, Boolean force) throws Exce // remove from index metadata not in DBMS for (String id : docs.keySet()) { - getSearchManager().delete("_id", id); + getSearchManager().delete(id); if (Log.isDebugEnabled(Geonet.DATA_MANAGER)) { Log.debug(Geonet.DATA_MANAGER, "- removed record (" + id + ") from index"); @@ -451,6 +452,7 @@ public synchronized void rebuildIndexXLinkedMetadata(final ServiceContext contex * Reindex all records in current selection. */ public synchronized void rebuildIndexForSelection(final ServiceContext context, + String bucket, boolean clearXlink) throws Exception { @@ -459,8 +461,8 @@ public synchronized void rebuildIndexForSelection(final ServiceContext context, UserSession session = context.getUserSession(); SelectionManager sm = SelectionManager.getManager(session); - synchronized (sm.getSelection("metadata")) { - for (Iterator iter = sm.getSelection("metadata").iterator(); + synchronized (sm.getSelection(bucket)) { + for (Iterator iter = sm.getSelection(bucket).iterator(); iter.hasNext(); ) { String uuid = (String) iter.next(); String id = getMetadataId(uuid); @@ -557,7 +559,7 @@ public boolean isIndexing() { public void indexMetadata(final List metadataIds) throws Exception { for (String metadataId : metadataIds) { - indexMetadata(metadataId, false); + indexMetadata(metadataId, false, null); } getSearchManager().forceIndexChanges(); @@ -566,7 +568,7 @@ public void indexMetadata(final List metadataIds) throws Exception { /** * TODO javadoc. */ - public void indexMetadata(final String metadataId, boolean forceRefreshReaders) throws Exception { + public void indexMetadata(final String metadataId, boolean forceRefreshReaders, ISearchManager searchManager) throws Exception { indexLock.lock(); try { if (waitForIndexing.contains(metadataId)) { @@ -762,7 +764,12 @@ public void indexMetadata(final String metadataId, boolean forceRefreshReaders) } moreFields.add(SearchManager.makeField(Geonet.IndexFieldNames.VALID, isValid, true, true)); } - getSearchManager().index(getSchemaManager().getSchemaDir(schema), md, metadataId, moreFields, metadataType, root, forceRefreshReaders); + + if (searchManager == null) { + getSearchManager().index(getSchemaManager().getSchemaDir(schema), md, metadataId, moreFields, metadataType, root, forceRefreshReaders); + } else { + searchManager.index(getSchemaManager().getSchemaDir(schema), md, metadataId, moreFields, metadataType, root, forceRefreshReaders); + } } catch (Exception x) { Log.error(Geonet.DATA_MANAGER, "The metadata document index with id=" + metadataId + " is corrupt/invalid - ignoring it. Error: " + x.getMessage(), x); fullMd = null; @@ -1283,7 +1290,7 @@ public String getNewVersion(String id) { */ public void setTemplate(final int id, final MetadataType type, final String title) throws Exception { setTemplateExt(id, type); - indexMetadata(Integer.toString(id), true); + indexMetadata(Integer.toString(id), true, null); } /** @@ -1304,7 +1311,7 @@ public void apply(@Nonnull Metadata metadata) { */ public void setHarvested(int id, String harvestUuid) throws Exception { setHarvestedExt(id, harvestUuid); - indexMetadata(Integer.toString(id), true); + indexMetadata(Integer.toString(id), true, null); } /** @@ -1436,7 +1443,7 @@ public void apply(Metadata entity) { } }); - indexMetadata(Integer.toString(metadataId), true); + indexMetadata(Integer.toString(metadataId), true, null); return rating; } @@ -1614,7 +1621,7 @@ public Metadata insertMetadata(ServiceContext context, Metadata newMetadata, Ele copyDefaultPrivForGroup(context, stringId, groupId, fullRightsForGroup); if (index) { - indexMetadata(stringId, forceRefreshReaders); + indexMetadata(stringId, forceRefreshReaders, null); } if (notifyChange) { @@ -1745,12 +1752,13 @@ public boolean existsMetadataUuid(String uuid) throws Exception { * Returns all the keywords in the system. */ public Element getKeywords() throws Exception { - Collection keywords = getSearchManager().getTerms("keyword"); + // TODO ES +// Collection keywords = getSearchManager().getTerms("keyword"); Element el = new Element("keywords"); - for (Object keyword : keywords) { - el.addContent(new Element("keyword").setText((String) keyword)); - } +// for (Object keyword : keywords) { +// el.addContent(new Element("keyword").setText((String) keyword)); +// } return el; } @@ -1828,7 +1836,7 @@ public synchronized Metadata updateMetadata(final ServiceContext context, final } finally { if (index) { //--- update search criteria - indexMetadata(metadataId, true); + indexMetadata(metadataId, true, null); } } // Return an up to date metadata record @@ -2138,7 +2146,7 @@ public synchronized void deleteMetadata(ServiceContext context, String metadataI } //--- update search criteria - getSearchManager().delete("_id", metadataId + ""); + getSearchManager().delete(metadataId + ""); // _entityManager.flush(); // _entityManager.clear(); } @@ -2152,7 +2160,7 @@ public synchronized void deleteMetadata(ServiceContext context, String metadataI public synchronized void deleteMetadataGroup(ServiceContext context, String metadataId) throws Exception { deleteMetadataFromDB(context, metadataId); //--- update search criteria - getSearchManager().deleteGroup("_id", metadataId + ""); + getSearchManager().delete(metadataId + ""); } /** @@ -2314,7 +2322,7 @@ private void transformMd(ServiceContext context, String metadataId, Element md, notifyMetadataChange(md, metadataId); //--- update search criteria - indexMetadata(metadataId, true); + indexMetadata(metadataId, true, null); } } @@ -2630,7 +2638,7 @@ public String getCurrentStatus(int metadataId) throws Exception { */ public MetadataStatus setStatus(ServiceContext context, int id, int status, ISODate changeDate, String changeMessage) throws Exception { MetadataStatus statusObject = setStatusExt(context, id, status, changeDate, changeMessage); - indexMetadata(Integer.toString(id), true); + indexMetadata(Integer.toString(id), true, null); return statusObject; } @@ -3221,7 +3229,7 @@ public int batchDeleteMetadataAndUpdateIndex(Specification specificati } // Remove records from the index - getSearchManager().delete("_id", Lists.transform(idsOfMetadataToDelete, new Function() { + getSearchManager().delete(Lists.transform(idsOfMetadataToDelete, new Function() { @Nullable @Override public String apply(@Nonnull Integer input) { @@ -3247,7 +3255,7 @@ private T getBean(Class requiredType) { return getApplicationContext().getBean(requiredType); } - private SearchManager getSearchManager() { + private ISearchManager getSearchManager() { return getBean(SearchManager.class); } diff --git a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java index e5868d04f32..a05e2fe9ebd 100644 --- a/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java @@ -111,7 +111,7 @@ public void run() { } try { - dataManager.indexMetadata(metadataId.toString(), false); + dataManager.indexMetadata(metadataId.toString(), false, null); } catch (Exception e) { Log.error(Geonet.INDEX_ENGINE, "Error indexing metadata '" + metadataId + "': " + e.getMessage() + "\n" + Util.getStackTrace(e)); diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java index b057b39ef86..6f1b3d640a2 100644 --- a/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java +++ b/core/src/main/java/org/fao/geonet/kernel/mef/Importer.java @@ -474,7 +474,7 @@ public void apply(@Nonnull final Metadata metadata) { Files.createDirectories(priDir); - dm.indexMetadata(metadataIdMap.get(index), true); + dm.indexMetadata(metadataIdMap.get(index), true, null); } // -------------------------------------------------------------------- diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java new file mode 100644 index 00000000000..26328fd1962 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -0,0 +1,582 @@ +/* + * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ + +package org.fao.geonet.kernel.search; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.gson.JsonElement; +import io.searchbox.client.JestResult; +import io.searchbox.core.Get; +import io.searchbox.core.Search; +import io.searchbox.core.SearchResult; +import jeeves.server.ServiceConfig; +import jeeves.server.UserSession; +import jeeves.server.context.ServiceContext; +import org.apache.commons.lang.StringUtils; +import org.fao.geonet.ApplicationContextHolder; +import org.fao.geonet.constants.Geonet; +import org.fao.geonet.domain.ISODate; +import org.fao.geonet.domain.Metadata; +import org.fao.geonet.domain.MetadataDataInfo_; +import org.fao.geonet.domain.MetadataType; +import org.fao.geonet.domain.Metadata_; +import org.fao.geonet.domain.Pair; +import org.fao.geonet.es.EsClient; +import org.fao.geonet.kernel.DataManager; +import org.fao.geonet.kernel.SelectionManager; +import org.fao.geonet.kernel.setting.SettingManager; +import org.fao.geonet.repository.MetadataRepository; +import org.fao.geonet.repository.SortUtils; +import org.fao.geonet.repository.specification.MetadataSpecs; +import org.fao.geonet.utils.Log; +import org.fao.geonet.utils.Xml; +import org.jdom.Element; +import org.jdom.JDOMException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.data.jpa.domain.Specifications; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class EsSearchManager implements ISearchManager { + public static final String ID = "id"; + public static final String DOC_TYPE = "docType"; + public static final String SCHEMA_INDEX_XSLT_FOLDER = "index-fields"; + public static final String SCHEMA_INDEX_XSTL_FILENAME = "index.xsl"; + public static final String FIELDNAME = "name"; + public static final String FIELDSTRING = "string"; + + @Value("${es.index.records}") + private String index = "records"; + + @Autowired + private EsClient client; + + public static Path getXSLTForIndexing(Path schemaDir) { + Path xsltForIndexing = schemaDir + .resolve(SCHEMA_INDEX_XSLT_FOLDER).resolve(SCHEMA_INDEX_XSTL_FILENAME); + if (!Files.exists(xsltForIndexing)) { + throw new RuntimeException(String.format( + "XSLT for schema indexing does not exist. Create file '%s'.", + xsltForIndexing.toString())); + } + return xsltForIndexing; + } + + private static void addMDFields(Element doc, Path schemaDir, Element metadata, String root) { + final Path styleSheet = getXSLTForIndexing(schemaDir); + try { + Element fields = Xml.transform(metadata, styleSheet); + /* Generates something like that: + + Contenu + */ + for (Element field : (List) fields.getChildren()) { + doc.addContent((Element) field.clone()); + } + } catch (Exception e) { + Log.error(Geonet.INDEX_ENGINE, + String.format("Indexing stylesheet contains errors: %s \n\t Marking the metadata as _indexingError=1 in index", + e.getMessage())); + doc.addContent(new Element(IndexFields.INDEXING_ERROR_FIELD).setText("1")); + doc.addContent(new Element(IndexFields.INDEXING_ERROR_MSG).setText("GNIDX-XSL||" + e.getMessage())); + StringBuilder sb = new StringBuilder(); + allText(metadata, sb); + doc.addContent(new Element("_text_").setText(sb.toString())); + } + } + + private static void allText(Element metadata, StringBuilder sb) { + String text = metadata.getText().trim(); + if (text.length() > 0) { + if (sb.length() > 0) + sb.append(" "); + sb.append(text); + } + @SuppressWarnings("unchecked") + List children = metadata.getChildren(); + for (Element aChildren : children) { + allText(aChildren, sb); + } + } + + private static void addMoreFields(Element doc, List fields) { + for (Element field : fields) { + doc.addContent(new Element(field.getAttributeValue(FIELDNAME)) + .setText(field.getAttributeValue(FIELDSTRING))); + } + } + +// public static String convertDate(Object date) { +// if (date != null) { +// return new ISODate((Date) date).toString(); +// } else { +// return null; +// } +// } + + public static Integer convertInteger(Object value) { + if (value == null) { + return null; + } else if (value instanceof Number) { + return ((Number) value).intValue(); + } else { + return Integer.valueOf(value.toString()); + } + } + + public static Element makeField(String name, String value) { + Element field = new Element("Field"); + field.setAttribute(EsSearchManager.FIELDNAME, name); + field.setAttribute(EsSearchManager.FIELDSTRING, value == null ? "" : value); + return field; + } + + /** + * Creates a new XML field for the Lucene index and add it to the document. + */ + public static void addField(Element xmlDoc, String name, String value, boolean store, boolean index) { + Element field = makeField(name, value); + xmlDoc.addContent(field); + } + + @Override + public void init(ServiceConfig handlerConfig) throws Exception { + } + + @Override + public void end() throws Exception { + } + + @Override + public MetaSearcher newSearcher(String stylesheetName) throws Exception { + //TODO + return null; + } + + private int commitInterval = 200; + private Map listOfDocumentsToIndex = new HashMap<>(); + + @Override + public void index(Path schemaDir, Element metadata, String id, List moreFields, + MetadataType metadataType, String root, boolean forceRefreshReaders) throws Exception { + + SettingManager settingManager = ApplicationContextHolder.get().getBean(SettingManager.class); + + Element docs = new Element("docs"); + Element allFields = new Element("doc"); + allFields.addContent(new Element(ID).setText(id)); + allFields.addContent(new Element(DOC_TYPE).setText("metadata")); + addMDFields(allFields, schemaDir, metadata, root); + addMoreFields(allFields, moreFields); + + + docs.addContent(allFields); + ObjectMapper mapper = new ObjectMapper(); + ObjectNode doc = documentToJson(docs).get(id); + + // ES does not allow a _source field + String catalog = doc.get("source").asText(); + doc.remove("source"); + doc.put("sourceCatalogue", catalog); + doc.put("territory", settingManager.getSiteName()); + doc.put("harvesterUuid", settingManager.getSiteId()); + doc.put("harvesterId", settingManager.getNodeURL()); + Map docListToIndex = new HashMap<>(); + docListToIndex.put(id, mapper.writeValueAsString(doc)); + listOfDocumentsToIndex.put(id, mapper.writeValueAsString(doc)); + if (listOfDocumentsToIndex.size() == commitInterval) { + sendDocumentsToIndex(); + } + } + + private void sendDocumentsToIndex() throws IOException { + synchronized (this) { + if (listOfDocumentsToIndex.size() > 0) { + System.out.println(String.format("Sending %d document to index", listOfDocumentsToIndex.size())); + client.bulkRequest(index, listOfDocumentsToIndex); + listOfDocumentsToIndex.clear(); + } + } + } + + /** + * Convert document to JSON. + */ + public Map documentToJson(Element xml) { + ObjectMapper mapper = new ObjectMapper(); + + List records = xml.getChildren("doc"); + Map listOfXcb = new HashMap<>(); + + // Loop on docs + for (int i = 0; i < records.size(); i++) { + Element record = records.get(i); + if (record != null && record instanceof Element) { + ObjectNode doc = mapper.createObjectNode(); + String id = null; + List elementNames = new ArrayList(); + List fields = record.getChildren(); + + // Loop on doc fields + for (int j = 0; j < fields.size(); j++) { + Element currentField = fields.get(j); + String name = currentField.getName(); + + if (!elementNames.contains(name)) { + // Register list of already processed names + elementNames.add(name); + + List nodeElements = record.getChildren(name); + boolean isArray = nodeElements.size() > 1; + + // Field starting with _ not supported in Kibana + // Those are usually GN internal fields + String propertyName = name.startsWith("_") ? + name.substring(1) : name; + + ArrayNode arrayNode = null; + if (isArray) { + arrayNode = doc.putArray(propertyName); + } + + // Group fields in array if needed + for (int k = 0; k < nodeElements.size(); k++) { + Element node = nodeElements.get(k); + + if (name.equals("id")) { + id = node.getTextNormalize(); + } + + if (name.equals("geom")) { + continue; + } + + if (isArray) { + arrayNode.add(node.getTextNormalize()); + } else if (name.equals("geojson")) { + doc.put("geom", node.getTextNormalize()); + } else if ( + // Skip some fields causing errors / TODO + !name.startsWith("conformTo_")) { + doc.put( + propertyName, + node.getTextNormalize()); + } + } + } + } + listOfXcb.put(id, doc); + } + } + return listOfXcb; + } + @Override + public void forceIndexChanges() throws IOException { + } + + @Override + public boolean rebuildIndex(ServiceContext context, boolean xlinks, + boolean reset, String bucket) throws Exception { + DataManager dataMan = context.getBean(DataManager.class); + MetadataRepository metadataRepository = context.getBean(MetadataRepository.class); + + if (reset) { + clearIndex(); + } + + if (StringUtils.isNotBlank(bucket)) { + ArrayList listOfIdsToIndex = new ArrayList(); + UserSession session = context.getUserSession(); + SelectionManager sm = SelectionManager.getManager(session); + + synchronized (sm.getSelection(bucket)) { + for (Iterator iter = sm.getSelection(bucket).iterator(); + iter.hasNext(); ) { + String uuid = (String) iter.next(); +// String id = dataMan.getMetadataId(uuid); + Metadata metadata = metadataRepository.findOneByUuid(uuid); + if (metadata != null) { + listOfIdsToIndex.add(metadata.getId() + ""); + } else { + System.out.println(String.format( + "Selection contains uuid '%s' not found in database", uuid)); + } + } + } + for(String id : listOfIdsToIndex) { + dataMan.indexMetadata(id + "", false, this); + } + sendDocumentsToIndex(); + } else { + final Specifications metadataSpec = + Specifications.where(MetadataSpecs.isType(MetadataType.METADATA)) + .or(MetadataSpecs.isType(MetadataType.TEMPLATE)); + final List metadataIds = metadataRepository.findAllIdsBy( + Specifications.where(metadataSpec) + ); + for(Integer id : metadataIds) { + dataMan.indexMetadata(id + "", false, this); + } + sendDocumentsToIndex(); + } + + return true; + } + + public void clearIndex() throws Exception { + SettingManager settingManager = ApplicationContextHolder.get().getBean(SettingManager.class); + client.deleteByQuery(index, + "harvesterUuid:\\\"" + settingManager.getSiteId() + "\\\""); + } + +// public void iterateQuery(SolrQuery params, final Consumer callback) throws IOException, SolrServerException { +// final MutableLong pos = new MutableLong(0); +// final MutableLong last = new MutableLong(1); +// while (pos.longValue() < last.longValue()) { +// params.setStart(pos.intValue()); +// client.queryAndStreamResponse(params, new StreamingResponseCallback() { +// @Override +// public void streamSolrDocument(SolrDocument doc) { +// pos.add(1); +// callback.accept(doc); +// } +// +// @Override +// public void streamDocListInfo(long numFound, long start, Float maxScore) { +// last.setValue(numFound); +// } +// }); +// } +// } + + @Override + public Map getDocsChangeDate() throws Exception { + String query = "{\"query\": {\"filtered\": {\"query_string\": \"*:*\"}}}"; + Search search = new Search.Builder(query).addIndex(index).addType(index).build(); + // TODO: limit to needed field +// params.setFields(ID, Geonet.IndexFieldNames.DATABASE_CHANGE_DATE); + SearchResult searchResult = client.getClient().execute(search); + +// final Map result = new HashMap<>(); +// iterateQuery(searchResult.getHits(), doc -> +// result.put(doc.getFieldValue(ID).toString(), +// convertDate(doc.getFieldValue(Geonet.IndexFieldNames.DATABASE_CHANGE_DATE)))); + return null; + } + + @Override + public ISODate getDocChangeDate(String mdId) throws Exception { + // TODO: limit to needed field + Get get = new Get.Builder(index, mdId).type(index).build(); + JestResult result = client.getClient().execute(get); + if (result != null) { + JsonElement date = + result.getJsonObject().get(Geonet.IndexFieldNames.DATABASE_CHANGE_DATE); + return date != null ? new ISODate(date.getAsString()) : null; + } else { + return null; + } + } + +// public SolrDocument getDocFieldValue(String query, String... field) throws IOException, SolrServerException { +// final SolrQuery params = new SolrQuery(query); +// params.setFilterQueries(DOC_TYPE + ":metadata"); +// params.setFields(field); +// QueryResponse response = client.query(params); +// final SolrDocumentList results = response.getResults(); +// if (results.size() == 0) { +// return null; +// } else { +// return results.get(0); +// } +// } +// +// public SolrDocumentList getDocsFieldValue(String query, String... field) throws IOException, SolrServerException { +// final SolrQuery params = new SolrQuery(query); +// params.setFilterQueries(DOC_TYPE + ":metadata"); +// params.setFields(field); +// QueryResponse response = client.query(params); +// return response.getResults(); +// } +// +// public List getDocsUuids(String query, Integer rows) throws IOException, SolrServerException { +// final SolrQuery solrQuery = new SolrQuery(query == null ? "*:*" : query); +// solrQuery.setFilterQueries(DOC_TYPE + ":metadata"); +// solrQuery.setFields(IndexFields.UUID); +// if (rows != null) { +// solrQuery.setRows(rows); +// } +// final List result = new ArrayList<>(); +// iterateQuery(solrQuery, doc -> +// result.add(doc.getFieldValue(IndexFields.UUID).toString())); +// return result; +// } + + @Override + public Set getDocsWithXLinks() throws Exception { +// final SolrQuery params = new SolrQuery("*:*"); +// params.setFilterQueries(DOC_TYPE + ":metadata"); +// params.setFilterQueries(Geonet.IndexFieldNames.HASXLINKS + ":1"); +// params.setFields(ID); +// Set result = new HashSet<>(); +// iterateQuery(params, +// doc -> result.add(convertInteger(doc.getFieldValue(ID)))); + return null; + } + + @Override + public void delete(String txt) throws Exception { + client.deleteByQuery(index, txt); +// client.commit(); + } + + @Override + public void delete(List txts) throws Exception { +// client.deleteById(txts); +// client.commit(); + } + + @Override + public void rescheduleOptimizer(Calendar beginAt, int interval) { + + } + + @Override + public void disableOptimizer() { + + } + + public int getNumDocs(String query) throws Exception { + if (StringUtils.isBlank(query)) { + query = "*:*"; + } + String searchQuery = "{\"query\": {\"filtered\": {\"query_string\": \"" + query + "\"}}}"; + Search search = new Search.Builder(searchQuery).addIndex(index).addType(index).build(); + SearchResult searchResult = client.getClient().execute(search); + return searchResult.getTotal(); + } + +// public List getDocFieldValues(String indexField, +// String query, +// boolean missing, +// Integer limit, +// String sort) throws IOException { +// final SolrQuery solrQuery = new SolrQuery(query == null ? "*:*" : query) +// .setFilterQueries(DOC_TYPE + ":metadata") +// .setRows(0) +// .setFacet(true) +// .setFacetMissing(missing) +// .setFacetLimit(limit != null ? limit : 1000) +// .setFacetSort(sort != null ? sort : "count") // or index +// .addFacetField(indexField); +// QueryResponse response = client.query(solrQuery); +// return response.getFacetField(indexField).getValues(); +// } +// +// public void updateRating(int metadataId, int newValue) throws IOException, SolrServerException { +// updateField(metadataId, Geonet.IndexFieldNames.RATING, newValue, "set"); +// } +// +// public void incrementPopularity(int metadataId) throws IOException, SolrServerException { +// //TODO: check that works +// updateField(metadataId, Geonet.IndexFieldNames.POPULARITY, 1, "inc"); +// } +// +// private void updateField(int metadataId, String fieldName, int newValue, String operator) throws IOException, SolrServerException { +// SolrInputDocument doc = new SolrInputDocument(); +// doc.addField(ID, metadataId); +// Map fieldModifier = new HashMap<>(1); +// fieldModifier.put(operator, newValue); +// doc.addField(fieldName, fieldModifier); +// client.add(doc); +// client.commit(); +// } + + public EsClient getClient() { + return client; + } + + /** + * Only for UTs + */ + void setClient(EsClient client) { + this.client = client; + } + + public List getDocs(String query, Integer start, Integer rows) throws IOException, JDOMException { + final List result = getDocIds(query, start, rows); + List xmlDocs = new ArrayList<>(result.size()); + MetadataRepository metadataRepository = ApplicationContextHolder.get().getBean(MetadataRepository.class); + for (String id : result) { + Metadata metadata = metadataRepository.findOne(id); + xmlDocs.add(metadata.getXmlData(false)); + } + return xmlDocs; + } + + public List getDocIds(String query, Integer start, Integer rows) throws IOException, JDOMException { +// final SolrQuery solrQuery = new SolrQuery(query == null ? "*:*" : query); +// solrQuery.setFilterQueries(DOC_TYPE + ":metadata"); +// solrQuery.setFields(SolrSearchManager.ID); +// if (start != null) { +// solrQuery.setStart(start); +// } +// if (rows != null) { +// solrQuery.setRows(rows); +// } +// QueryResponse response = client.query(solrQuery); +// SolrDocumentList results = response.getResults(); +// List idList = new ArrayList<>(results.size()); +// for (SolrDocument document : results) { +// idList.add(document.getFieldValue(SolrSearchManager.ID).toString()); +// } +// return idList; + return null; + } + + public List getAllDocs(String query) throws Exception { + int hitsNumber = getNumDocs(query); + return getDocs(query, 0, hitsNumber); + } + + public List getAllDocIds(String query) throws Exception { + int hitsNumber = getNumDocs(query); + return getDocIds(query, 0, hitsNumber); + } + +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java new file mode 100644 index 00000000000..6388b3579e7 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/ISearchManager.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ + +package org.fao.geonet.kernel.search; + +import org.fao.geonet.domain.ISODate; +import org.fao.geonet.domain.MetadataType; +import org.jdom.Element; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import jeeves.server.ServiceConfig; +import jeeves.server.context.ServiceContext; + +/** + * Base interface for the search (Lucene or Solr). + */ +public interface ISearchManager { + void init(ServiceConfig handlerConfig) throws Exception; + + void end() throws Exception; + + MetaSearcher newSearcher(String stylesheetName) throws Exception; + + /** + * Indexes a metadata record. + * + * @param forceRefreshReaders if true then block all searches until they can obtain a up-to-date + * reader + */ + void index(Path schemaDir, Element metadata, String id, List moreFields, + MetadataType metadataType, String root, boolean forceRefreshReaders) + throws Exception; + + /** + * Force the index to wait until all changes are processed and the next reader obtained will get + * the latest data. + */ + void forceIndexChanges() throws IOException; + + + /** + * Rebuilds the Lucene index. If xlink or from selection parameters are defined, reindex a + * subset of record. Otherwise reindex all records. + * + * @param xlinks Search all docs with XLinks, clear the XLinks cache and index all + * records found. + * @param bucket Reindex all records from selection bucket. + */ + boolean rebuildIndex(ServiceContext context, + boolean xlinks, + boolean reset, + String bucket) throws Exception; + + Map getDocsChangeDate() throws Exception; + + ISODate getDocChangeDate(String mdId) throws Exception; + + Set getDocsWithXLinks() throws Exception; + + /** + * deletes a document. + */ + void delete(String txt) throws Exception; + + /** + * deletes a list of documents. + */ + void delete(List txts) throws Exception; + + void rescheduleOptimizer(Calendar beginAt, int interval) throws Exception; + + void disableOptimizer() throws Exception; +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/IndexFields.java b/core/src/main/java/org/fao/geonet/kernel/search/IndexFields.java new file mode 100644 index 00000000000..76f2901efa4 --- /dev/null +++ b/core/src/main/java/org/fao/geonet/kernel/search/IndexFields.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ + +package org.fao.geonet.kernel.search; + +/** + * Names of fields in the Lucene index. + * + * @author heikki doeleman + */ +public class IndexFields { + + /** + * Groups allowed to view. + */ + public static final String _OP0 = "_op0"; + /** + * Groups allowed to download. + */ + public static final String _OP1 = "_op1"; + /** + * Groups allowed to edit. + */ + public static final String _OP2 = "_op2"; + /** + * Groups allowed to be notified. + */ + public static final String _OP3 = "_op3"; + /** + * Groups allowed "dynamic". + */ + public static final String _OP5 = "_op5"; + /** + * Groups allowed to feature. + */ + public static final String _OP6 = "_op6"; + + public static final String ABSTRACT = "abstract"; + public static final String ANY = "any"; + public static final String CAT = "_cat"; + public static final String CHANGE_DATE = "changeDate"; + public static final String CREATE_DATE = "createDate"; + public static final String CREDIT = "credit"; + public static final String DATAPARAM = "dataparam"; + public static final String DENOMINATOR_FROM = "denominatorFrom"; + public static final String DENOMINATOR_TO = "denominatorTo"; + public static final String DENOMINATOR = "denominator"; + public static final String DIGITAL = "digital"; + public static final String DOWNLOAD = "download"; + public static final String DUMMY = "_dummy"; + public static final String EAST = "eastBL"; + public static final String GROUP_OWNER = "_groupOwner"; + public static final String ID = "_id"; + public static final String INSPIRE_ANNEX = "inspireannex"; + public static final String INSPIRE_CAT = "inspirecat"; + public static final String INSPIRE_THEME = "inspiretheme"; + public static final String IS_TEMPLATE = "_isTemplate"; + public static final String KEYWORD = "keyword"; + public static final String METADATA_STANDARD_NAME = "metadataStandardName"; + public static final String NORTH = "northBL"; + public static final String OPERATESON = "operatesOn"; + public static final String ORG_NAME = "orgName"; + public static final String OWNER = "_owner"; + public static final String PAPER = "paper"; + public static final String PARENTUUID = "parentUuid"; + public static final String PROTOCOL = "protocol"; + public static final String PUBLICATION_DATE = "publicationDate"; + public static final String REVISION_DATE = "revisionDate"; + public static final String SCHEMA = "_schema"; + public static final String SERVICE_TYPE = "serviceType"; + public static final String SOURCE = "_source"; + public static final String SOUTH = "southBL"; + public static final String SPATIALREPRESENTATIONTYPE = "spatialRepresentationType"; + public static final String SUBJECT = "subject"; + public static final String TAXON = "taxon:name"; + public static final String TEMPORALEXTENT_BEGIN = "tempExtentBegin"; + public static final String TEMPORALEXTENT_END = "tempExtentEnd"; + public static final String TITLE = "title"; + public static final String ALT_TITLE = "altTitle"; + public static final String TOPIC_CATEGORY = "topicCat"; + public static final String TYPE = "type"; + public static final String UUID = "_uuid"; + public static final String VALID = "_valid"; + public static final String WEST = "westBL"; + public static final String RECORD_IDENTIFIER = "metadataIdentifier"; + public static final String RESOURCE_IDENTIFIER = "resourceIdentifier"; + public static final String HAS_ATOM = "has_atom"; + public static final String PARENT_RECORD_UUID = "parent"; + public static final String RECORD_OPERATE_ON = "recordOperateOn"; + public static final String RESOURCE_TITLE = "resourceTitle"; + public static final String RESOURCE_ABSTRACT = "resourceAbstract"; + public static final String RESOURCE_TYPE = "resourceType"; + public static final String ROOT = "_root"; + public static final String INDEXING_ERROR_MSG = "_indexingErrorMsg"; + public static final String INDEXING_ERROR_FIELD = "_indexingError"; +} diff --git a/core/src/main/java/org/fao/geonet/kernel/search/SearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/SearchManager.java index ec8e0a159a0..6871beea330 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/SearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/SearchManager.java @@ -52,6 +52,7 @@ import javax.annotation.PreDestroy; +import jeeves.server.ServiceConfig; import org.apache.commons.lang.StringUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.core.KeywordAnalyzer; @@ -140,7 +141,7 @@ /** * Indexes metadata using Lucene. */ -public class SearchManager { +public class SearchManager implements ISearchManager { public static final String INDEXING_ERROR_FIELD = "_indexingError"; private static final String INDEXING_ERROR_MSG = "_indexingErrorMsg"; private static final String SEARCH_STYLESHEETS_DIR_PATH = "xml/search"; @@ -556,6 +557,11 @@ public void initNonStaticData(boolean logAsynch, boolean logSpatialObject, Strin _luceneOptimizerManager = new LuceneOptimizerManager(this, settingInfo); } + @Override + public void init(ServiceConfig handlerConfig) throws Exception { + + } + @PreDestroy public void end() throws Exception { // the tracker is closed when the ApplicationContext is closed. @@ -564,6 +570,11 @@ public void end() throws Exception { _luceneOptimizerManager.shutdown(); } + @Override + public MetaSearcher newSearcher(String stylesheetName) throws Exception { + return null; + } + /** * TODO javadoc. */ @@ -615,7 +626,8 @@ public synchronized void disableOptimizer() throws Exception { /** * TODO javadoc. */ - public synchronized void rescheduleOptimizer(Calendar optimizerBeginAt, int optimizerInterval) throws Exception { + public synchronized void rescheduleOptimizer( + Calendar optimizerBeginAt, int optimizerInterval) throws Exception { _luceneOptimizerManager.reschedule(optimizerBeginAt, optimizerInterval); } @@ -919,6 +931,16 @@ public Set getDocsWithXLinks() throws Exception { } } + @Override + public void delete(String txt) throws Exception { + delete("_id", txt); + } + + @Override + public void delete(List txts) throws Exception { + delete("_id", txts); + } + public ISODate getDocChangeDate(String mdId) throws Exception { Query query = new TermQuery(new Term(LuceneIndexField.ID, mdId)); try (final IndexAndTaxonomy indexReader = getIndexReader(Geonet.DEFAULT_LANGUAGE, -1)) { @@ -1232,12 +1254,12 @@ private void setupIndex(boolean rebuild) throws Exception { * * @param xlinks Search all docs with XLinks, clear the XLinks cache and index all * records found. - * @param fromSelection Reindex all records from selection. + * @param bucket Reindex all records from selection bucket. */ public boolean rebuildIndex(ServiceContext context, boolean xlinks, boolean reset, - boolean fromSelection) throws Exception { + String bucket) throws Exception { DataManager dataMan = context.getBean(DataManager.class); LuceneIndexLanguageTracker _tracker = context.getBean(LuceneIndexLanguageTracker.class); try { @@ -1246,8 +1268,8 @@ public boolean rebuildIndex(ServiceContext context, setupIndex(false); } } - if (fromSelection) { - dataMan.rebuildIndexForSelection(context, xlinks); + if (StringUtils.isNotBlank(bucket)) { + dataMan.rebuildIndexForSelection(context, bucket, xlinks); } else if (xlinks) { dataMan.rebuildIndexXLinkedMetadata(context); } else { diff --git a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java index ecc978e7509..34b7407fec5 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/index/IndexingTask.java @@ -25,8 +25,6 @@ import jeeves.server.context.ServiceContext; import jeeves.server.dispatchers.ServiceManager; -import jeeves.transaction.TransactionManager; -import jeeves.transaction.TransactionTask; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.constants.Geonet; @@ -75,7 +73,7 @@ private void indexRecords() { for (Integer metadataIdentifier : metadataIdentifiers) { try { - _dataManager.indexMetadata(String.valueOf(metadataIdentifier), false); + _dataManager.indexMetadata(String.valueOf(metadataIdentifier), false, null); } catch (Exception e) { Log.error(Geonet.INDEX_ENGINE, "Indexing task / An error happens indexing the metadata " + metadataIdentifier + ". Error: " + e.getMessage(), e); diff --git a/core/src/main/java/org/fao/geonet/services/thumbnail/Set.java b/core/src/main/java/org/fao/geonet/services/thumbnail/Set.java index f8e3a5927d8..6732bf59ddf 100644 --- a/core/src/main/java/org/fao/geonet/services/thumbnail/Set.java +++ b/core/src/main/java/org/fao/geonet/services/thumbnail/Set.java @@ -153,7 +153,7 @@ public Response serviceSpecificExec(HttpServletRequest request, dataMan.setThumbnail(context, id, type.equals("small"), file.getOriginalFilename(), false); } - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); return new Response(id, dataMan.getNewVersion(id)); } @@ -209,7 +209,7 @@ public Element execOnHarvest( saveThumbnail(scaling, file, type, dataDir, scalingDir, scalingFactor, dataMan, id, context); //----------------------------------------------------------------------- - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); Element response = new Element("Response"); response.addContent(new Element("id").setText(id)); diff --git a/core/src/main/resources/config-spring-geonetwork.xml b/core/src/main/resources/config-spring-geonetwork.xml index 2466ae328e4..ee5e6018ab5 100644 --- a/core/src/main/resources/config-spring-geonetwork.xml +++ b/core/src/main/resources/config-spring-geonetwork.xml @@ -58,6 +58,7 @@ + fileIds, ServiceCont dataMan.setOperation(context, id, "" + ReservedGroup.all.getId(), ReservedOperation.view); } - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); fileIds.add(uuid); diff --git a/es/README.md b/es/README.md index 48063b4ab9e..9c3dc64bba9 100644 --- a/es/README.md +++ b/es/README.md @@ -26,6 +26,7 @@ cd es mvn install -Pes-download mvn exec:exec -Des-start curl -X PUT http://localhost:9200/features -d @config/features.json +curl -X PUT http://localhost:9200/records -d @config/records.json ``` To delete your index: diff --git a/es/config/features.json b/es/config/features.json index f74eef9da1b..abb45b62f23 100644 --- a/es/config/features.json +++ b/es/config/features.json @@ -81,7 +81,16 @@ "search_analyzer": "keyword" } } - } + }, + { + "stringType": { + "match": "_op*", + "mapping": { + "type": "keyword", + "fielddata": true + } + } + }, ], "properties": { "id": { diff --git a/es/config/records.json b/es/config/records.json new file mode 100644 index 00000000000..2e670f66c67 --- /dev/null +++ b/es/config/records.json @@ -0,0 +1,1267 @@ +{ + "settings": { + "index": { + "analysis": { + "analyzer": { + "synInspireAnnexes": { + "tokenizer": "keyword", + "filter": [ + "synInspireThemes", + "synInspireAnnexes", + "keepInspireAnnexes" + ] + }, + "synInspireThemes": { + "tokenizer": "keyword", + "filter": [ + "synInspireThemes" + ] + }, + "keepInspireServiceTypes": { + "tokenizer": "keyword", + "filter": [ + "lowercase", "keepInspireServiceTypes" + ] + }, + "keepInspireAnnexes": { + "tokenizer": "keyword", + "filter": [ + "lowercase", "keepInspireAnnexes" + ] + }, + "keepInspireThemes": { + "tokenizer": "keyword", + "filter": [ + "lowercase", "keepInspireThemes" + ] + }, + "synCheckpoint": { + "tokenizer": "keyword", + "filter": [ + "synCheckpoint" + ] + } + }, + "filter": { + "keepInspireServiceTypes": { + "type": "keep", + "keep_words": [ + "discovery", + "view", + "download", + "invoke", + "transformation", + "other" + ] + }, + "keepInspireAnnexes": { + "type": "keep", + "keep_words": [ + "i", + "ii", + "iii" + ] + }, + "keepInspireThemes": { + "type": "keep", + "keep_words": [ + "Coordinate reference systems", + "Elevation", + "Land cover", + "Orthoimagery", + "Geology", + "Statistical units", + "Buildings", + "Soil", + "Land use", + "Human health and safety", + "Utility and governmental services", + "Geographical grid systems", + "Environmental monitoring facilities", + "Production and industrial facilities", + "Agricultural and aquaculture facilities", + "Population distribution — demography", + "Area management/restriction/regulation zones and reporting units", + "Natural risk zones", + "Atmospheric conditions", + "Meteorological geographical features", + "Oceanographic geographical features", + "Sea regions", + "Geographical names", + "Bio-geographical regions", + "Habitats and biotopes", + "Species distribution", + "Energy resources", + "Mineral resources", + "Administrative units", + "Addresses", + "Cadastral parcels", + "Transport networks", + "Hydrography", + "Protected sites" + ] + }, + "synInspireAnnexes": { + "type": "synonym", + "tokenizer": "keyword", + "ignore_case": true, + "synonyms": [ + "Coordinate reference systems => I", + "Elevation => II", + "Land cover => II", + "Orthoimagery => II", + "Geology => II", + "Statistical units => III", + "Buildings => III", + "Soil => III", + "Land use => III", + "Human health and safety => III", + "Utility and governmental services => III", + "Geographical grid systems => I", + "Environmental monitoring facilities => III", + "Production and industrial facilities => III", + "Agricultural and aquaculture facilities => III", + "Population distribution — demography => III", + "Area management/restriction/regulation zones and reporting units => III", + "Natural risk zones => III", + "Atmospheric conditions => III", + "Meteorological geographical features => III", + "Oceanographic geographical features => III", + "Sea regions => III", + "Geographical names => I", + "Bio-geographical regions => III", + "Habitats and biotopes => III", + "Species distribution => III", + "Energy resources => III", + "Mineral resources => III", + "Administrative units => I", + "Addresses => I", + "Cadastral parcels => I", + "Transport networks => I", + "Hydrography => I", + "Protected sites => I" + ] + }, + "synInspireThemes": { + "type": "synonym", + "tokenizer": "keyword", + "ignore_case": true, + "synonyms": [ + "Referenskoordinatsystem => Coordinate reference systems", + "Referenčni koordinatni sistemi => Coordinate reference systems", + "Súradnicové referenčné systémy => Coordinate reference systems", + "Sisteme de coordonate de referință => Coordinate reference systems", + "Sisteme de coordonate de referinţă => Coordinate reference systems", + "Systemy odniesienia za pomocą współrzędnych => Coordinate reference systems", + "Sistemas de referencia => Coordinate reference systems", + "Systemen voor verwijzing door middel van coördinaten => Coordinate reference systems", + "Sistemi ta' referenza ta' koordinati => Coordinate reference systems", + "Koordinātu atskaites sistēmas => Coordinate reference systems", + "Koordinačių atskaitos sistemos => Coordinate reference systems", + "Sistemi di coordinate => Coordinate reference systems", + "Koordinátarendszerek => Coordinate reference systems", + "Référentiels de coordonnées => Coordinate reference systems", + "Koordinaattijärjestelmät => Coordinate reference systems", + "Sistemas de coordenadas de referencia => Coordinate reference systems", + "Koordinaatsüsteemid => Coordinate reference systems", + "Συστήματα συντεταγμένων => Coordinate reference systems", + "Koordinatenreferenzsysteme => Coordinate reference systems", + "Координатни справочни системи => Coordinate reference systems", + "Koordinatsystemer => Coordinate reference systems", + "Souřadnicové referenční systémy => Coordinate reference systems", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/1 => Coordinate reference systems", + "Höjd => Elevation", + "Altitude => Elevation", + "Elevaţie => Elevation", + "Elevație => Elevation", + "Digitalni model višin => Elevation", + "Výška => Elevation", + "Ukształtowanie terenu => Elevation", + "Hoogte => Elevation", + "Elevazzjoni => Elevation", + "Augstums => Elevation", + "Aukštis => Elevation", + "Elevazione => Elevation", + "Korkeus => Elevation", + "Domborzat => Elevation", + "Υψομετρία => Elevation", + "Elevaciones => Elevation", + "Kõrgused => Elevation", + "Höhe => Elevation", + "Højde => Elevation", + "Nadmořská výška => Elevation", + "Релеф => Elevation", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/10 => Elevation", + "Landtäcke => Land cover", + "Pokrovnost tal => Land cover", + "Ocupação do solo => Land cover", + "Krajinná pokrývka (land cover) => Land cover", + "Acoperire terestră => Land cover", + "Bodemgebruik => Land cover", + "Użytkowanie terenu => Land cover", + "Zemes virsma => Land cover", + "Kopertura ta' l-art => Land cover", + "Žemės danga => Land cover", + "A felszín borítása => Land cover", + "Occupation des terres => Land cover", + "Copertura del suolo => Land cover", + "Cubierta terrestre => Land cover", + "Maakate => Land cover", + "Maanpeite => Land cover", + "Κάλυψη γης => Land cover", + "Bodenbedeckung => Land cover", + "Krajinný pokryv => Land cover", + "Arealdække => Land cover", + "Земна покривка => Land cover", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/11 => Land cover", + "Ortofoto => Orthoimagery", + "Ortoimagini => Orthoimagery", + "Ortometria => Orthoimagery", + "Sporządzanie ortoobrazów => Orthoimagery", + "Ortoimagens => Orthoimagery", + "Orthobeeldvorming => Orthoimagery", + "Orthoimagery => Orthoimagery", + "Ortofotogrāfija => Orthoimagery", + "Ortofotografinis vaizdavimas => Orthoimagery", + "Orto immagini => Orthoimagery", + "Ortofotók => Orthoimagery", + "Ortho-imagerie => Orthoimagery", + "Ortoimágenes => Orthoimagery", + "Ortoilmakuvat => Orthoimagery", + "Ortokujutised => Orthoimagery", + "Ορθοφωτογραφία => Orthoimagery", + "Orthofotografie => Orthoimagery", + "Ortofotosnímky => Orthoimagery", + "Ортоизображение => Orthoimagery", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/12 => Orthoimagery", + "Geologia => Geology", + "Geologie => Geology", + "Geológia => Geology", + "Geologija => Geology", + "Geologi => Geology", + "Ġeologija => Geology", + "Ģeoloģija => Geology", + "Földtan => Geology", + "Géologie => Geology", + "Geología => Geology", + "Geoloogia => Geology", + "Γεωλογία => Geology", + "Геология => Geology", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/13 => Geology", + "Unidades estatísticas => Statistical units", + "Unităţi statistice => Statistical units", + "Štatistické jednotky => Statistical units", + "Statistični okoliši => Statistical units", + "Statistiska enheter => Statistical units", + "Statistische eenheden => Statistical units", + "Jednostki statystyczne => Statistical units", + "Statistiniai vienetai => Statistical units", + "Unitajiet statistiċi => Statistical units", + "Unități statistice => Statistical units", + "Statistikas vienības => Statistical units", + "Unità statistiche => Statistical units", + "Tilastoyksiköt => Statistical units", + "Unités statistiques => Statistical units", + "Statisztikai egységek => Statistical units", + "Unidades estadísticas => Statistical units", + "Statistilised üksused => Statistical units", + "Statistische Einheiten => Statistical units", + "Στατιστικές μονάδες => Statistical units", + "Statistické jednotky => Statistical units", + "Statistiske enheder => Statistical units", + "Статистически единици => Statistical units", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/14 => Statistical units", + "Stavbe => Buildings", + "Byggnader => Buildings", + "Clădiri => Buildings", + "Stavby => Buildings", + "Edifícios => Buildings", + "Budynki => Buildings", + "Ēkas => Buildings", + "Bini => Buildings", + "Gebouwen => Buildings", + "Κτίρια => Buildings", + "Edificios => Buildings", + "Ehitised => Buildings", + "Rakennukset => Buildings", + "Pastatai => Buildings", + "Edifici => Buildings", + "Épületek => Buildings", + "Bâtiments => Buildings", + "Сгради => Buildings", + "Budovy => Buildings", + "Bygninger => Buildings", + "Gebäude => Buildings", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/15 => Buildings", + "Tla => Soil", + "Mark => Soil", + "Soluri => Soil", + "Gleba => Soil", + "Pôda => Soil", + "Solo => Soil", + "Augsne => Soil", + "Bodem => Soil", + "Ħamrija => Soil", + "Suolo => Soil", + "Dirvožemis => Soil", + "Talaj => Soil", + "Sols => Soil", + "Maaperä => Soil", + "Pinnas => Soil", + "Suelo => Soil", + "Έδαφος => Soil", + "Boden => Soil", + "Jord => Soil", + "Půda => Soil", + "Почва => Soil", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/16 => Soil", + "Markanvändning => Land use", + "Namenska raba tal => Land use", + "Zagospodarowanie przestrzenne => Land use", + "Uso do solo => Land use", + "Utilizarea terenului => Land use", + "Využitie územia => Land use", + "Zemes izmantošana => Land use", + "Użu ta' l-art => Land use", + "Landgebruik => Land use", + "Földhasználat => Land use", + "Žemės naudojimas => Land use", + "Utilizzo del territorio => Land use", + "Maakasutus => Land use", + "Maankäyttö => Land use", + "Usage des sols => Land use", + "Χρήσεις γης => Land use", + "Uso del suelo => Land use", + "Ползване на земята => Land use", + "Využití území => Land use", + "Arealanvendelse => Land use", + "Bodennutzung => Land use", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/17 => Land use", + "Människors hälsa och säkerhet => Human health and safety", + "Zdravje in varnost prebivalstva => Human health and safety", + "Ľudské zdravie a bezpečnosť => Human health and safety", + "Zdrowie i bezpieczeństwo ludzi => Human health and safety", + "Sănătate şi siguranţă umană => Human health and safety", + "Sănătate și siguranță umană => Human health and safety", + "Saúde humana e segurança => Human health and safety", + "Žmonių sveikata ir sauga => Human health and safety", + "Cilvēku veselība un drošība => Human health and safety", + "Is-saħħa u s-siġurtà tal-bniedem => Human health and safety", + "Menselijke gezondheid en veiligheid => Human health and safety", + "Salute umana e sicurezza => Human health and safety", + "Emberi egészség és biztonság => Human health and safety", + "Santé et sécurité des personnes => Human health and safety", + "Väestön terveys ja turvallisuus => Human health and safety", + "Salud y seguridad humanas => Human health and safety", + "Inimeste tervis ja ohutus => Human health and safety", + "Sikkerhed => Human health and safety", + "Gesundheit und Sicherheit => Human health and safety", + "Ανθρώπινη υγεία και ασφάλεια => Human health and safety", + "Здраве и безопасност на човека => Human health and safety", + "Lidské zdraví a bezpečnost => Human health and safety", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/18 => Human health and safety", + "Allmännyttiga och offentliga tjänster => Utility and governmental services", + "Komunalne in javne storitve => Utility and governmental services", + "Servicii de utilitate publică şi servicii publice => Utility and governmental services", + "Servicii de utilitate publică și servicii publice => Utility and governmental services", + "Verejné a štátne služby => Utility and governmental services", + "Usługi użyteczności publicznej i służby państwowe => Utility and governmental services", + "Serviços de utilidade pública e do Estado => Utility and governmental services", + "Nutsdiensten en overheidsdiensten => Utility and governmental services", + "Komunalinės įmonės ir valstybės tarnybos => Utility and governmental services", + "Servizzi ta' utilità u tal-gvern => Utility and governmental services", + "Komunālie un valsts dienesti => Utility and governmental services", + "Servizi di pubblica utilità e servizi amministrativi => Utility and governmental services", + "Services d'utilité publique et services publics => Utility and governmental services", + "Közüzemi és közszolgáltatások => Utility and governmental services", + "Yleishyödylliset ja muut julkiset palvelut => Utility and governmental services", + "Servicios de utilidad pública y estatales => Utility and governmental services", + "Kommunaal- ja riiklikud teenused => Utility and governmental services", + "Offentlig forsyningsvirksomhed og offentlige tjenesteydelser => Utility and governmental services", + "Versorgungswirtschaft und staatliche Dienste => Utility and governmental services", + "Комунално-битови и обществени услуги => Utility and governmental services", + "Επιχειρήσεις κοινής ωφελείας και κρατικές υπηρεσίες => Utility and governmental services", + "Veřejné služby a služby veřejné správy => Utility and governmental services", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/19 => Utility and governmental services", + "Geografska koordinatna mreža => Geographical grid systems", + "Geografiska rutnätssystem => Geographical grid systems", + "Sisteme de caroiaj geografic => Geographical grid systems", + "Geografické systémy sietí => Geographical grid systems", + "Systemy siatek geograficznych => Geographical grid systems", + "Sistemas de quadrículas geográficas => Geographical grid systems", + "Sistemi ta' grilji ġeografiċi => Geographical grid systems", + "Geografisch rastersysteem => Geographical grid systems", + "Ģeogrāfisko koordinātu tīklu sistēmas => Geographical grid systems", + "Geografinio tinklelio sistemos => Geographical grid systems", + "Sistemi di griglie geografiche => Geographical grid systems", + "Systèmes de maillage géographique => Geographical grid systems", + "Földrajzi rácsrendszerek => Geographical grid systems", + "Paikannusruudustot => Geographical grid systems", + "Sistema de cuadrículas geográficas => Geographical grid systems", + "Geograafilised ruutvõrgud => Geographical grid systems", + "Συστήματα γεωγραφικού καννάβου => Geographical grid systems", + "Geografiske kvadratnetsystemer => Geographical grid systems", + "Geografische Gittersysteme => Geographical grid systems", + "Географски координатни системи => Geographical grid systems", + "Zeměpisné soustavy souřadnicových sítí => Geographical grid systems", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/2 => Geographical grid systems", + "Anläggningar för miljöövervakning => Environmental monitoring facilities", + "Naprave in objekti za monitoring okolja => Environmental monitoring facilities", + "Zariadenia na monitorovanie životného prostredia => Environmental monitoring facilities", + "Instalações de monitorização do ambiente => Environmental monitoring facilities", + "Milieubewakingsvoorzieningen => Environmental monitoring facilities", + "Urządzenia do monitorowania środowiska => Environmental monitoring facilities", + "Instalaţii de supraveghere a mediului => Environmental monitoring facilities", + "Instalații de supraveghere a mediului => Environmental monitoring facilities", + "Faċilitajiet ta' monitoraġġ ambjentali => Environmental monitoring facilities", + "Aplinkos stebėsenos priemonės => Environmental monitoring facilities", + "Vides monitoringa iekārtas => Environmental monitoring facilities", + "Környezetvédelmi monitoringlétesítmények => Environmental monitoring facilities", + "Impianti di monitoraggio ambientale => Environmental monitoring facilities", + "Installations de suivi environnemental => Environmental monitoring facilities", + "Keskkonnaseirerajatised => Environmental monitoring facilities", + "Ympäristön tilan seurantalaitteet => Environmental monitoring facilities", + "Instalaciones de observación del medio ambiente => Environmental monitoring facilities", + "Εγκαταστάσεις παρακολούθησης του περιβάλλοντος => Environmental monitoring facilities", + "Miljøovervågningsfaciliteter => Environmental monitoring facilities", + "Umweltüberwachung => Environmental monitoring facilities", + "Zařízení pro sledování životního prostředí => Environmental monitoring facilities", + "Съоръжения за управление на околната среда => Environmental monitoring facilities", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/20 => Environmental monitoring facilities", + "Produktions- och industrianläggningar => Production and industrial facilities", + "Instalaţii de producţie şi industriale => Production and industrial facilities", + "Instalații de producție și industriale => Production and industrial facilities", + "Proizvodni in industrijski objekti in naprave => Production and industrial facilities", + "Výrobné a priemyselné zariadenia => Production and industrial facilities", + "Faciliteiten voor productie en industrie => Production and industrial facilities", + "Instalações industriais e de produção => Production and industrial facilities", + "Obiekty produkcyjne i przemysłowe => Production and industrial facilities", + "Ražošanas un rūpniecības iekārtas => Production and industrial facilities", + "Faċilitajiet ta' produzzjoni u industrijali => Production and industrial facilities", + "Produzione e impianti industriali => Production and industrial facilities", + "Gamybos ir pramonės įrenginiai => Production and industrial facilities", + "Tuotanto- ja teollisuuslaitokset => Production and industrial facilities", + "Lieux de production et sites industriels => Production and industrial facilities", + "Termelő és ipari létesítmények => Production and industrial facilities", + "Tootmis- ja tööstusrajatised => Production and industrial facilities", + "Instalaciones de producción e industriales => Production and industrial facilities", + "Εγκαταστάσεις παραγωγής και βιομηχανικές εγκαταστάσεις => Production and industrial facilities", + "Produktions- und Industrieanlagen => Production and industrial facilities", + "Výrobní a průmyslová zařízení => Production and industrial facilities", + "Produktions- og industrifaciliteter => Production and industrial facilities", + "Производствени и промишлени съоръжения => Production and industrial facilities", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/21 => Production and industrial facilities", + "Jordbruks- och vattenbruksanläggningar => Agricultural and aquaculture facilities", + "Objekti in naprave za kmetijstvo in ribogojstvo => Agricultural and aquaculture facilities", + "Poľnohospodárske zariadenia a zariadenia akvakultúry => Agricultural and aquaculture facilities", + "Instalações agrícolas e aquícolas => Agricultural and aquaculture facilities", + "Instalaţii agricole şi pentru acvacultură => Agricultural and aquaculture facilities", + "Instalații agricole și pentru acvacultură => Agricultural and aquaculture facilities", + "Obiekty rolnicze oraz akwakultury => Agricultural and aquaculture facilities", + "Faċilitajiet agrikoli u ta' l-akwakultura => Agricultural and aquaculture facilities", + "Faciliteiten voor landbouw en aquacultuur => Agricultural and aquaculture facilities", + "Lauksaimniecības un akvakultūras iekārtas => Agricultural and aquaculture facilities", + "Žemės ūkio ir akvakultūros infrastruktūra => Agricultural and aquaculture facilities", + "Impianti agricoli e di acquacoltura => Agricultural and aquaculture facilities", + "Γεωργικές εγκαταστάσεις και εγκαταστάσεις υδατοκαλλιέργειας => Agricultural and aquaculture facilities", + "Mezőgazdasági és akvakultúra-ágazati létesítmények => Agricultural and aquaculture facilities", + "Installations agricoles et aquacoles => Agricultural and aquaculture facilities", + "Maatalous- ja vesiviljelylaitokset => Agricultural and aquaculture facilities", + "Põllumajandus- ja vesiviljelusrajatised => Agricultural and aquaculture facilities", + "Instalaciones agrícolas y de acuicultura => Agricultural and aquaculture facilities", + "Селскостопански и водностопански съоръжения => Agricultural and aquaculture facilities", + "Zemědělská a akvakulturní zařízení => Agricultural and aquaculture facilities", + "Landwirtschaftliche Anlagen und Aquakulturanlagen => Agricultural and aquaculture facilities", + "Landbrugs- og akvakulturanlæg => Agricultural and aquaculture facilities", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/22 => Agricultural and aquaculture facilities", + "Befolkningsfördelning – demografi => Population distribution — demography", + "Porazdelitev prebivalstva – demografski podatki => Population distribution — demography", + "Repartizarea populaţiei – demografie => Population distribution — demography", + "Repartizarea populației – demografie => Population distribution — demography", + "Rozmiestnenie obyvateľstva – demografia => Population distribution — demography", + "Distribuição da população — demografia => Population distribution — demography", + "Spreiding van de bevolking — demografie => Population distribution — demography", + "Rozmieszczenie ludności – demografia => Population distribution — demography", + "Iedzīvotāju sadalījums – demogrāfija => Population distribution — demography", + "Distribuzzjoni tal-popolazzjoni – demografija => Population distribution — demography", + "Distribuzione della popolazione — demografia => Population distribution — demography", + "Gyventojų pasiskirstymas – demografija => Population distribution — demography", + "Répartition de la population — démographie => Population distribution — demography", + "A népesség eloszlása – demográfia => Population distribution — demography", + "Elanikkonna jaotumine – demograafia => Population distribution — demography", + "Väestöjakauma – demografia => Population distribution — demography", + "Distribución de la población — demografía => Population distribution — demography", + "Κατανομή πληθυσμού — δημογραφία => Population distribution — demography", + "Rozložení obyvatelstva – demografie => Population distribution — demography", + "Verteilung der Bevölkerung — Demografie => Population distribution — demography", + "Befolkningsfordeling — demografi => Population distribution — demography", + "Разпределение на населението — демография => Population distribution — demography", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/23 => Population distribution — demography", + "Spravované/obmedzené/regulované zóny a jednotky podávajúce správy => Area management/restriction/regulation zones and reporting units", + "Områden med särskild förvaltning/begränsningar/reglering samt enheter för rapportering => Area management/restriction/regulation zones and reporting units", + "Območja upravljanja/zaprta območja/regulirana območja in poročevalske enote => Area management/restriction/regulation zones and reporting units", + "Zone de administrare/restricţie/reglementare şi unităţi de raportare => Area management/restriction/regulation zones and reporting units", + "Zone de administrare/restricție/reglementare și unități de raportare => Area management/restriction/regulation zones and reporting units", + "Gebiedsbeheer, gebieden waar beperkingen gelden, gereguleerde gebieden en rapportage-eenheden => Area management/restriction/regulation zones and reporting units", + "Zonas de gestão/restrição/regulamentação e unidades de referência => Area management/restriction/regulation zones and reporting units", + "Gospodarowanie obszarem/strefy ograniczone/regulacyjne oraz jednostki sprawozdawcze => Area management/restriction/regulation zones and reporting units", + "Apgabala pārvaldības/ierobežojumu/reglamentētas zonas un ziņošanas vienības => Area management/restriction/regulation zones and reporting units", + "Zone sottoposte a gestione/limitazioni/regolamentazione e unità con obbligo di comunicare dati => Area management/restriction/regulation zones and reporting units", + "Amministrazzjoni ta' żoni/restrizzjoni/żoni regolati u unitajiet ta' rapportar => Area management/restriction/regulation zones and reporting units", + "Tvarkomos teritorijos, ribojamos ir reglamentuojamos zonos bei vienetai, už kuriuos atsiskaitoma => Area management/restriction/regulation zones and reporting units", + "Területgazdálkodási/-korlátozási/-szabályozási övezetek és adatszolgáltató egységek => Area management/restriction/regulation zones and reporting units", + "Zones de gestion, de restriction ou de réglementation et unités de déclaration => Area management/restriction/regulation zones and reporting units", + "Aluesuunnittelun, rajoitusten ja sääntelyn piiriin kuuluvat alueet ja raportointiyksiköt => Area management/restriction/regulation zones and reporting units", + "Üldplaneering/piirangu-/reguleeritud tsoonid ja aruandlusüksused => Area management/restriction/regulation zones and reporting units", + "Zonas sujetas a ordenación, a restricciones o reglamentaciones y unidades de notificación => Area management/restriction/regulation zones and reporting units", + "Bewirtschaftungsgebiete/Schutzgebiete/geregelte Gebiete und Berichterstattungseinheiten => Area management/restriction/regulation zones and reporting units", + "Ζώνες διαχείρισης/περιορισμού/ρύθμισης εκτάσεων και μονάδες αναφοράς => Area management/restriction/regulation zones and reporting units", + "Forvaltede og regulerede områder samt områder med brugsbegrænsning og indberetningsenheder => Area management/restriction/regulation zones and reporting units", + "Správní oblasti/chráněná pásma/regulovaná území a jednotky podávající hlášení => Area management/restriction/regulation zones and reporting units", + "Управление на територията/ограничени/регулирани зони и отчетни единици => Area management/restriction/regulation zones and reporting units", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/24 => Area management/restriction/regulation zones and reporting units", + "Območja nevarnosti naravnih nesreč => Natural risk zones", + "Naturliga riskområden => Natural risk zones", + "Zóny prírodného rizika => Natural risk zones", + "Zone de risc natural => Natural risk zones", + "Zonas de risco natural => Natural risk zones", + "Strefy zagrożenia naturalnego => Natural risk zones", + "Żoni ta' riskju naturali => Natural risk zones", + "Gebieden met natuurrisico'es => Natural risk zones", + "Gebieden met natuurrisico's => Natural risk zones", + "Dabas apdraudējuma zonas => Natural risk zones", + "Gamtinių pavojų zonos => Natural risk zones", + "Természeti kockázati zónák => Natural risk zones", + "Zone a rischio naturale => Natural risk zones", + "Zones à risque naturel => Natural risk zones", + "Luonnonriskialueet => Natural risk zones", + "Looduslikud ohutsoonid => Natural risk zones", + "Zonas de riesgos naturales => Natural risk zones", + "Ζώνες φυσικών κινδύνων => Natural risk zones", + "Природни рискови зони => Natural risk zones", + "Områder med naturlige risici => Natural risk zones", + "Gebiete mit naturbedingten Risiken => Natural risk zones", + "Oblasti ohrožené přírodními riziky => Natural risk zones", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/25 => Natural risk zones", + "Ozračje => Atmospheric conditions", + "Atmosfäriska förhållanden => Atmospheric conditions", + "Condiţii atmosferice => Atmospheric conditions", + "Condiții atmosferice => Atmospheric conditions", + "Atmosférické podmienky => Atmospheric conditions", + "Warunki atmosferyczne => Atmospheric conditions", + "Condições atmosféricas => Atmospheric conditions", + "Atmosferische omstandigheden => Atmospheric conditions", + "Kondizzjonijiet atmosferiċi => Atmospheric conditions", + "Atmosfēras apstākļi => Atmospheric conditions", + "Condizioni atmosferiche => Atmospheric conditions", + "Atmosferos sąlygos => Atmospheric conditions", + "Conditions atmosphériques => Atmospheric conditions", + "Légköri viszonyok => Atmospheric conditions", + "Atmosfääritingimused => Atmospheric conditions", + "Ilmakehän tila => Atmospheric conditions", + "Condiciones atmosféricas => Atmospheric conditions", + "Atmosphärische Bedingungen => Atmospheric conditions", + "Ατμοσφαιρικές συνθήκες => Atmospheric conditions", + "Stav ovzduší => Atmospheric conditions", + "Atmosfæriske forhold => Atmospheric conditions", + "Атмосферни условия => Atmospheric conditions", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/26 => Atmospheric conditions", + "Meteorološke značilnosti => Meteorological geographical features", + "Geografiska meteorologiska förhållanden => Meteorological geographical features", + "Meteorologické geografické prvky => Meteorological geographical features", + "Características geometeorológicas => Meteorological geographical features", + "Caracteristici geografice meteorologice => Meteorological geographical features", + "Warunki meteorologiczno-geograficzne => Meteorological geographical features", + "Meteorologische geografische kenmerken => Meteorological geographical features", + "Karatteristiċi ġeografiċi meteoroloġiċi => Meteorological geographical features", + "Meteoroloģiski ģeogrāfiskie raksturlielumi => Meteorological geographical features", + "Meteorologinės geografinės sąlygos => Meteorological geographical features", + "Elementi geografici meteorologici => Meteorological geographical features", + "Meteorológiai földrajzi jellemzők => Meteorological geographical features", + "Ilmaston maantieteelliset ominaispiirteet => Meteorological geographical features", + "Caractéristiques géographiques météorologiques => Meteorological geographical features", + "Meteoroloogilis-geograafilised tunnusjooned => Meteorological geographical features", + "Μετεωρολογικά γεωγραφικά χαρακτηριστικά => Meteorological geographical features", + "Aspectos geográficos de carácter meteorológico => Meteorological geographical features", + "Meteorologisch-geografische Kennwerte => Meteorological geographical features", + "Meteorologisk-geografiske forhold => Meteorological geographical features", + "Метеорологични географски характеристики => Meteorological geographical features", + "Zeměpisné meteorologické prvky => Meteorological geographical features", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/27 => Meteorological geographical features", + "Geografiska oceanografiska förhållanden => Oceanographic geographical features", + "Oceanogeografske značilnosti => Oceanographic geographical features", + "Características oceanográficas => Oceanographic geographical features", + "Caracteristici geografice oceanografice => Oceanographic geographical features", + "Oceánografické geografické prvky => Oceanographic geographical features", + "Warunki oceanograficzno-geograficzne => Oceanographic geographical features", + "Karatteristiċi ġeografiċi oċeanografiċi => Oceanographic geographical features", + "Oceanografische geografische kenmerken => Oceanographic geographical features", + "Okeanogrāfiski ģeogrāfiskie raksturlielumi => Oceanographic geographical features", + "Elementi geografici oceanografici => Oceanographic geographical features", + "Okeanografinės geografinės sąlygos => Oceanographic geographical features", + "Oceanográfiai földrajzi jellemzők => Oceanographic geographical features", + "Caractéristiques géographiques océanographiques => Oceanographic geographical features", + "Merentutkimuksen maantieteelliset ominaispiirteet => Oceanographic geographical features", + "Okeanograafilis-geograafilised tunnusjooned => Oceanographic geographical features", + "Rasgos geográficos oceanográficos => Oceanographic geographical features", + "Oceanografiske/geografiske forhold => Oceanographic geographical features", + "Ozeanografisch-geografische Kennwerte => Oceanographic geographical features", + "Ωκεανογραφικά γεωγραφικά χαρακτηριστικά => Oceanographic geographical features", + "Zeměpisné oceánografické prvky => Oceanographic geographical features", + "Океанографски географски характеристики => Oceanographic geographical features", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/28 => Oceanographic geographical features", + "Havsområden => Sea regions", + "Morske regije => Sea regions", + "Morské regióny => Sea regions", + "Regiuni maritime => Sea regions", + "Zeegebieden => Sea regions", + "Regiony morskie => Sea regions", + "Regiões marinhas => Sea regions", + "Reġjuni tal-baħar => Sea regions", + "Jūru reģioni => Sea regions", + "Jūrų regionai => Sea regions", + "Regioni marine => Sea regions", + "Tengeri régiók => Sea regions", + "Régions maritimes => Sea regions", + "Merepiirkonnad => Sea regions", + "Merialueet => Sea regions", + "Regiones marinas => Sea regions", + "Θαλάσσιες περιοχές => Sea regions", + "Mořské oblasti => Sea regions", + "Havområder => Sea regions", + "Meeresregionen => Sea regions", + "Морски региони => Sea regions", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/29 => Sea regions", + "Zemljepisna imena => Geographical names", + "Geografiska namn => Geographical names", + "Zemepisné názvy => Geographical names", + "Denumiri geografice => Geographical names", + "Nazwy geograficzne => Geographical names", + "Toponímia => Geographical names", + "Toponīmi => Geographical names", + "Ismijiet ġeografiċi => Geographical names", + "Geografische namen => Geographical names", + "Dénominations géographiques => Geographical names", + "Földrajzi nevek => Geographical names", + "Nomi geografici => Geographical names", + "Geografiniai pavadinimai => Geographical names", + "Paikannimet => Geographical names", + "Geograafilised nimed => Geographical names", + "Nombres geográficos => Geographical names", + "Τοπωνύμια => Geographical names", + "Geografische Bezeichnungen => Geographical names", + "Географски наименования => Geographical names", + "Zeměpisné názvy => Geographical names", + "Stednavne => Geographical names", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/3 => Geographical names", + "Biogeografiska regioner => Bio-geographical regions", + "Biogeografske regije => Bio-geographical regions", + "Biogeografické regióny => Bio-geographical regions", + "Regiuni biogeografice => Bio-geographical regions", + "Regiony biogeograficzne => Bio-geographical regions", + "Regiões biogeográficas => Bio-geographical regions", + "Biogeografische gebieden => Bio-geographical regions", + "Biogeografiniai regionai => Bio-geographical regions", + "Bioģeogrāfiskie reģioni => Bio-geographical regions", + "Reġjuni bijo-ġeografiċi => Bio-geographical regions", + "Regioni biogeografiche => Bio-geographical regions", + "Biogeográfiai régiók => Bio-geographical regions", + "Régions biogéographiques => Bio-geographical regions", + "Biomaantieteelliset alueet => Bio-geographical regions", + "Bio-geograafilised piirkonnad => Bio-geographical regions", + "Regiones biogeográficas => Bio-geographical regions", + "Βιογεωγραφικές περιοχές => Bio-geographical regions", + "Biogeografische Regionen => Bio-geographical regions", + "Biogeografiske regioner => Bio-geographical regions", + "Bioregiony => Bio-geographical regions", + "Биогеографски региони => Bio-geographical regions", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/30 => Bio-geographical regions", + "Naturtyper och biotoper => Habitats and biotopes", + "Habitati in biotopi => Habitats and biotopes", + "Habitaty a biotopy => Habitats and biotopes", + "Habitate şi biotopuri => Habitats and biotopes", + "Habitate și biotopuri => Habitats and biotopes", + "Habitats e biótopos => Habitats and biotopes", + "Siedliska i obszary przyrodniczo jednorodne => Habitats and biotopes", + "Habitats en biotopen => Habitats and biotopes", + "L-abitati naturali u l-bijotopi => Habitats and biotopes", + "Dzīvotnes un biotopi => Habitats and biotopes", + "Buveinės ir biotopai => Habitats and biotopes", + "Habitat e biotopi => Habitats and biotopes", + "Élőhelyek és biotópok => Habitats and biotopes", + "Habitats et biotopes => Habitats and biotopes", + "Elinympäristöt ja biotoopit => Habitats and biotopes", + "Elupaigad ja biotoobid => Habitats and biotopes", + "Hábitats y biotopos => Habitats and biotopes", + "Ενδιαιτήματα και βιότοποι => Habitats and biotopes", + "Lebensräume und Biotope => Habitats and biotopes", + "Levesteder og biotoper => Habitats and biotopes", + "Местообитания и биотопи => Habitats and biotopes", + "Stanoviště a biotopy => Habitats and biotopes", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/31 => Habitats and biotopes", + "Arters utbredning => Species distribution", + "Porazdelitev vrst => Species distribution", + "Výskyt druhov => Species distribution", + "Repartizarea speciilor => Species distribution", + "Distribuição das espécies => Species distribution", + "Rozmieszczenie gatunków => Species distribution", + "Spreiding van soorten => Species distribution", + "Distribuzzjoni ta' l-ispeċi => Species distribution", + "Sugu izplatība => Species distribution", + "Rūšių pasiskirstymas => Species distribution", + "Distribuzione delle specie => Species distribution", + "A fajok megoszlása => Species distribution", + "Répartition des espèces => Species distribution", + "Lajien levinneisyys => Species distribution", + "Liikide jaotumine => Species distribution", + "Distribución de las especies => Species distribution", + "Κατανομή ειδών => Species distribution", + "Verteilung der Arten => Species distribution", + "Rozložení druhů => Species distribution", + "Artsfordeling => Species distribution", + "Разпределение на видовете => Species distribution", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/32 => Species distribution", + "Zdroje energie => Energy resources", + "Energetski viri => Energy resources", + "Energiresurser => Energy resources", + "Recursos energéticos => Energy resources", + "Resurse energetice => Energy resources", + "Zasoby energetyczne => Energy resources", + "Energiebronnen => Energy resources", + "Enerģijas resursi => Energy resources", + "Riżorsi ta' enerġija => Energy resources", + "Energijos ištekliai => Energy resources", + "Risorse energetiche => Energy resources", + "Energiaforrások => Energy resources", + "Sources d'énergie => Energy resources", + "Energiavarat => Energy resources", + "Energiaressursid => Energy resources", + "Ενεργειακοί πόροι => Energy resources", + "Energiequellen => Energy resources", + "Energiressourcer => Energy resources", + "Energetické zdroje => Energy resources", + "Енергийни източници => Energy resources", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/33 => Energy resources", + "Mineralni viri => Mineral resources", + "Mineralfyndigheter => Mineral resources", + "Zdroje nerastných surovín => Mineral resources", + "Resurse minerale => Mineral resources", + "Recursos minerais => Mineral resources", + "Zasoby mineralne => Mineral resources", + "Minerale bronnen => Mineral resources", + "Riżorsi minerali => Mineral resources", + "Derīgo izrakteņu resursi => Mineral resources", + "Naudingosios iškasenos => Mineral resources", + "Risorse minerarie => Mineral resources", + "Ásványi nyersanyagok => Mineral resources", + "Ressources minérales => Mineral resources", + "Mineraalivarat => Mineral resources", + "Maavarad => Mineral resources", + "Recursos minerales => Mineral resources", + "Ορυκτοί πόροι => Mineral resources", + "Mineralische Bodenschätze => Mineral resources", + "Mineralressourcer => Mineral resources", + "Nerostné suroviny => Mineral resources", + "Минерални ресурси => Mineral resources", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/34 => Mineral resources", + "Upravne enote => Administrative units", + "Administrativa enheter => Administrative units", + "Správne jednotky => Administrative units", + "Unităţi administrative => Administrative units", + "Unități administrative => Administrative units", + "Jednostki administracyjne => Administrative units", + "Unidades administrativas => Administrative units", + "Administratieve eenheden => Administrative units", + "Unitajiet amministrattivi => Administrative units", + "Administratīvas vienības => Administrative units", + "Közigazgatási egységek => Administrative units", + "Unità amministrative => Administrative units", + "Administraciniai vienetai => Administrative units", + "Unités administratives => Administrative units", + "Hallinnolliset yksiköt => Administrative units", + "Haldusüksused => Administrative units", + "Διοικητικές ενότητες => Administrative units", + "Verwaltungseinheiten => Administrative units", + "Administrative enheder => Administrative units", + "Административни единици => Administrative units", + "Správní jednotky => Administrative units", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/4 => Administrative units", + "Adrese => Addresses", + "Adresy => Addresses", + "Naslovi => Addresses", + "Adresser => Addresses", + "Endereços => Addresses", + "Indirizzi => Addresses", + "Adressen => Addresses", + "Adresai => Addresses", + "Adreses => Addresses", + "Címek => Addresses", + "Adresses => Addresses", + "Osoitteet => Addresses", + "Direcciones => Addresses", + "Aadressid => Addresses", + "Διευθύνσεις => Addresses", + "Адреси => Addresses", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/5 => Addresses", + "Fastighetsområden => Cadastral parcels", + "Katastrske parcele => Cadastral parcels", + "Katastrálne parcely => Cadastral parcels", + "Parcele cadastrale => Cadastral parcels", + "Pakketti katastali => Cadastral parcels", + "Kadastrale percelen => Cadastral parcels", + "Działki katastralne => Cadastral parcels", + "Parcelas cadastrais => Cadastral parcels", + "Kadastrāli zemes gabali => Cadastral parcels", + "Kadastro sklypai => Cadastral parcels", + "Parcelle catastali => Cadastral parcels", + "Parcelles cadastrales => Cadastral parcels", + "Kataszteri parcellák => Cadastral parcels", + "Γεωτεμάχια κτηματολογίου => Cadastral parcels", + "Parcelas catastrales => Cadastral parcels", + "Katastriüksused => Cadastral parcels", + "Kiinteistöt => Cadastral parcels", + "Кадастрални парцели => Cadastral parcels", + "Katastrální parcely => Cadastral parcels", + "Matrikulære parceller => Cadastral parcels", + "Flurstücke/Grundstücke (Katasterparzellen) => Cadastral parcels", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/6 => Cadastral parcels", + "Transportnät => Transport networks", + "Prometna omrežja => Transport networks", + "Dopravné siete => Transport networks", + "Reţele de transport => Transport networks", + "Rețele de transport => Transport networks", + "Redes de transporte => Transport networks", + "Sieci transportowe => Transport networks", + "Transporta tīkli => Transport networks", + "Networks tat-trasport => Transport networks", + "Vervoersnetwerken => Transport networks", + "Transporto tinklai => Transport networks", + "Közlekedési hálózatok => Transport networks", + "Reti di trasporto => Transport networks", + "Liikenneverkot => Transport networks", + "Réseaux de transport => Transport networks", + "Transpordivõrgud => Transport networks", + "Δίκτυα μεταφορών => Transport networks", + "Verkehrsnetze => Transport networks", + "Transportnet => Transport networks", + "Транспортни мрежи => Transport networks", + "Dopravní sítě => Transport networks", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/7 => Transport networks", + "Hidrografija => Hydrography", + "Hydrografi => Hydrography", + "Hydrografia => Hydrography", + "Hidrografia => Hydrography", + "Hidrografie => Hydrography", + "Hydrografie => Hydrography", + "Idrografija => Hydrography", + "Hidrogrāfija => Hydrography", + "Hydrographie => Hydrography", + "Idrografia => Hydrography", + "Vízrajz => Hydrography", + "Hüdrograafia => Hydrography", + "Hidrografía => Hydrography", + "Υδρογραφία => Hydrography", + "Gewässernetz => Hydrography", + "Hydrograf => Hydrography", + "Vodopis => Hydrography", + "Хидрография => Hydrography", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/8 => Hydrography", + "Zavarovana območja => Protected sites", + "Skyddade områden => Protected sites", + "Zone protejate => Protected sites", + "Chránené územia => Protected sites", + "Obszary chronione => Protected sites", + "Sítios protegidos => Protected sites", + "Beschermde gebieden => Protected sites", + "Siti protetti => Protected sites", + "Saugomos teritorijos => Protected sites", + "Aizsargājamas teritorijas => Protected sites", + "Suojellut alueet => Protected sites", + "Sites protégés => Protected sites", + "Védett helyek => Protected sites", + "Kaitsealused kohad => Protected sites", + "Lugares protegidos => Protected sites", + "Προστατευόμενες τοποθεσίες => Protected sites", + "Beskyttede lokaliteter => Protected sites", + "Schutzgebiete => Protected sites", + "Chráněná území => Protected sites", + "Защитени обекти => Protected sites", + "http://rdfdata.eionet.europa.eu/inspirethemes/themes/9 => Protected sites" + ] + }, + "synCheckpoint": { + "type": "synonym", + "tokenizer": "keyword", + "ignore_case": true, + "synonyms": [ + "No charge => Free", + "Distribution cost charge => Cost charge", + "Collection cost charge => Cost charge", + "Commercial cost charge => Commercial cost", + "Cost charge depends on intented use and category of users => Cost charge", + "Cost charge depends on intended use and category of users => Cost charge", + "Restricted => Restricted", + "Accessible under moratorium => Partially restricted", + "Unrestricted => Unrestricted", + "<15mn (on-line download) => High response", + "Less than 3 hours => Medium response", + "Less than 24 hours => Medium response", + "Less than 1 week => Medium response", + "More than 1 week => Low response", + "Not documented => Unknown", + "Unknown => Unknown", + "Order form/envoice => Manual", + "Order form/invoice => Manual", + "On-line downloading services => Partial INSPIRE function", + "On-line discovery+viewing + downloading services => Full INSPIRE function", + "On-line discovery + viewing + downloading services => Full INSPIRE function", + "On-line discovery + viewing + downloading + advanced services (High resolution viewing,3D viewing services, Processing/Mapping services...) => Full INSPIRE function" + ] + } + } + } + } + }, + "mappings": { + "_default_": { + "dynamic_templates": [ + { + "codelist": { + "match": "codelist_*", + "mapping": { + "type": "keyword" + } + } + }, + { + "org": { + "match": "*Org*", + "mapping": { + "type": "keyword" + } + } + }, + { + "contact": { + "match": "*contact*", + "mapping": { + "type": "keyword" + } + } + }, + { + "date": { + "match": "*Date*", + "mapping": { + "type": "date" + } + } + }, + { + "dateMonth": { + "match": "*Month*", + "mapping": { + "type": "keyword" + } + } + }, + { + "dateYear": { + "match": "*Year*", + "mapping": { + "type": "short" + } + } + }, + { + "measure": { + "match": "measure_*", + "mapping": { + "type": "keyword" + } + } + }, + { + "operationAllowed": { + "match": "op*", + "mapping": { + "type": "keyword" + } + } + }, + { + "validationStatus": { + "match": "valid*", + "mapping": { + "type": "keyword" + } + } + }, + { + "synInspireAnnexesType": { + "match": "inspireAnnex*", + "mapping": { + "type": "string", + "fielddata": true, + "analyzer": "synInspireAnnexes", + "search_analyzer": "keyword" + } + } + }, + { + "synInspireThemeType": { + "match": "inspireTheme*", + "mapping": { + "type": "string", + "fielddata": true, + "analyzer": "synInspireThemes", + "search_analyzer": "keyword" + } + } + }, + { + "inspireServiceType": { + "match": "inspireServiceType", + "mapping": { + "type": "string", + "fielddata": true, + "analyzer": "keepInspireServiceTypes", + "search_analyzer": "keyword" + } + } + } + ] + }, + "records": { + "properties": { + "accessConstraints": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "coordinateSystem": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "dateStamp": { + "type": "date" + }, + "document": { + "type": "text" + }, + "documentStandard": { + "type": "keyword" + }, + "documentType": { + "type": "keyword" + }, + "geom": { + "type": "text" + }, + "harvesterId": { + "type": "keyword" + }, + "harvesterUuid": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "hasInspireTheme": { + "type": "boolean" + }, + "hasOverview": { + "type": "boolean" + }, + "isValid": { + "type": "boolean" + }, + "isSchemaValid": { + "type": "boolean" + }, + "isAboveThreshold": { + "type": "boolean" + }, + "inspireThemeFirst_syn": { + "type": "keyword" + }, + "inspireTheme_syn": { + "type": "keyword" + }, + "isOpenData": { + "type": "keyword" + }, + "lineage": { + "type": "text" + }, + "link": { + "type": "text" + }, + "linkProtocol": { + "type": "keyword" + }, + "linkUrl": { + "type": "text" + }, + "mainLanguage": { + "type": "keyword" + }, + "metadataIdentifier": { + "type": "keyword" + }, + "numberOfInspireTheme": { + "type": "short" + }, + "otherConstraints": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "overviewUrl": { + "type": "text" + }, + "pointOfTruthURL": { + "type": "text" + }, + "recordOperateOn": { + "type": "keyword" + }, + "resolutionDistance": { + "type": "keyword" + }, + "resolutionScaleDenominator": { + "type": "keyword" + }, + "resourceAbstract": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "resourceAltTitle": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "resourceCredit": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "resourceLanguage": { + "type": "keyword" + }, + "resourceTitle": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "resourceType": { + "type": "keyword" + }, + "serviceType": { + "type": "keyword" + }, + "spatialRepresentationType": { + "type": "keyword" + }, + "standardName": { + "type": "keyword" + }, + "tag": { + "type": "keyword" + }, + "territory": { + "type": "keyword" + }, + "topic": { + "type": "keyword" + }, + "validDate": { + "type": "date", + "format": "date_hour_minute_second" + }, + "schemaValidDate": { + "type": "date", + "format": "date_hour_minute_second" + }, + "useLimitation": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "groupPublished": { + "type": "keyword" + }, + "changeDate": { + "type": "keyword" + }, + "createDate": { + "type": "keyword" + }, + "groupOwner": { + "type": "keyword" + }, + "hasxlinks": { + "type": "boolean" + }, + "isTemplate": { + "type": "keyword" + }, + "isHarvested": { + "type": "boolean" + }, + "popularity": { + "type": "short" + }, + "rating": { + "type": "short" + }, + "status": { + "type": "short" + }, + "owner": { + "type": "keyword" + }, + "xlink": { + "type": "keyword" + }, + "root": { + "type": "keyword" + } + } + } + } +} diff --git a/es/es-core/pom.xml b/es/es-core/pom.xml new file mode 100644 index 00000000000..e30e1630f4e --- /dev/null +++ b/es/es-core/pom.xml @@ -0,0 +1,40 @@ + + + + es + org.geonetwork-opensource + 3.3.0-SNAPSHOT + + 4.0.0 + + es-core + + + ${project.groupId} + common + ${project.version} + + + + + + io.searchbox + jest + + + + org.springframework + spring-web + + + + + diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/DeleteByQuery.java b/es/es-core/src/main/java/org/fao/geonet/es/DeleteByQuery.java similarity index 97% rename from workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/DeleteByQuery.java rename to es/es-core/src/main/java/org/fao/geonet/es/DeleteByQuery.java index 1cd57046eea..23c32b537d6 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/DeleteByQuery.java +++ b/es/es-core/src/main/java/org/fao/geonet/es/DeleteByQuery.java @@ -1,4 +1,4 @@ -package org.fao.geonet.harvester.wfsfeatures; +package org.fao.geonet.es; import io.searchbox.action.AbstractMultiTypeActionBuilder; import io.searchbox.action.GenericResultAbstractAction; diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsClient.java b/es/es-core/src/main/java/org/fao/geonet/es/EsClient.java similarity index 85% rename from workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsClient.java rename to es/es-core/src/main/java/org/fao/geonet/es/EsClient.java index dd04a40437e..3c72f93ef10 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsClient.java +++ b/es/es-core/src/main/java/org/fao/geonet/es/EsClient.java @@ -1,4 +1,27 @@ -package org.fao.geonet.harvester.wfsfeatures.worker; +/* + * Copyright (C) 2001-2015 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ + +package org.fao.geonet.es; import io.searchbox.client.JestClient; import io.searchbox.client.JestClientFactory; @@ -7,13 +30,12 @@ import io.searchbox.core.Bulk; import io.searchbox.core.BulkResult; import io.searchbox.core.Index; -import org.fao.geonet.harvester.wfsfeatures.DeleteByQuery; import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Value; import java.io.IOException; import java.util.Iterator; import java.util.Map; -import java.util.logging.Logger; /** @@ -21,13 +43,12 @@ */ public class EsClient implements InitializingBean { - Logger logger = Logger.getLogger(WFSHarvesterRouteBuilder.LOGGER_NAME); - private static EsClient instance; private JestClient client; + + @Value("${es.url}") private String serverUrl; - private String collection; private String username; private String password; @@ -59,7 +80,8 @@ public void afterPropertiesSet() throws Exception { instance = this; } } else { - throw new Exception(String.format("No ES client URL defined in %s. " + throw new Exception(String.format( + "No ES client URL defined in %s. " + "Check bean configuration.", this.serverUrl)); } } @@ -91,14 +113,6 @@ public EsClient setPassword(String password) { return this; } - public String getCollection() { - return collection; - } - - public void setCollection(String collection) { - this.collection = collection; - } - public boolean bulkRequest(String index, Map docs) throws IOException { boolean success = true; Bulk.Builder bulk = new Bulk.Builder() @@ -189,7 +203,9 @@ public String deleteByQuery(String index, String query) throws Exception { if (result.isSucceeded()) { return String.format("Record removed. %s.", result.getJsonString()); } else { - return String.format("Error during removal. Errors is '%s'.", result.getErrorMessage()); + throw new IOException(String.format( + "Error during removal. Errors is '%s'.", result.getErrorMessage() + )); } // // Search search = new Search.TemplateBuilder(searchQuery) diff --git a/es/pom.xml b/es/pom.xml index 1c329ba422c..7dddb859288 100644 --- a/es/pom.xml +++ b/es/pom.xml @@ -9,6 +9,7 @@ 4.0.0 es + pom @@ -77,6 +78,9 @@ + + es-core + elasticsearch diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java index 32ed0f95fc1..888e132c0f4 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/arcsde/ArcSDEHarvester.java @@ -22,17 +22,13 @@ //============================================================================== package org.fao.geonet.kernel.harvest.harvester.arcsde; -import com.google.common.base.Function; import jeeves.server.context.ServiceContext; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.HttpClientBuilder; import org.fao.geonet.Logger; import org.fao.geonet.arcgis.ArcSDEConnection; import org.fao.geonet.constants.Geonet; -import org.fao.geonet.constants.Params; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.Metadata; import org.fao.geonet.domain.MetadataType; @@ -373,7 +369,7 @@ private void updateMetadata(Element xml, String id, GroupMapper localGroups, fin dataMan.flush(); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); } /** @@ -417,7 +413,7 @@ private String addMetadata(Element xml, String uuid, String schema, GroupMapper aligner.addPrivileges(id, params.getPrivileges(), localGroups, dataMan, context, log); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java index 92e8093f818..2300870d32c 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/csw/Aligner.java @@ -303,7 +303,7 @@ private void addMetadata(RecordInfo ri) throws Exception { addPrivileges(id, params.getPrivileges(), localGroups, dataMan, context, log); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, Math.random() < 0.01, null); result.addedMetadata++; } @@ -359,7 +359,7 @@ private void updateMetadata(RecordInfo ri, String id) throws Exception { dataMan.flush(); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, Math.random() < 0.01, null); result.updatedMetadata++; } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java index 21e3329d2d1..2771d30fe23 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/fragment/FragmentHarvester.java @@ -412,7 +412,7 @@ private void createSubtemplate(String schema, Element md, String uuid) throws Ex addPrivileges(id, params.privileges, localGroups, dataMan, context, log); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); dataMan.flush(); @@ -586,7 +586,7 @@ private void update(String id, Element template, boolean doExt) throws Exception dataMan.setHarvestedExt(iId, params.uuid, Optional.of(harvestUri)); } - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); dataMan.flush(); } @@ -634,7 +634,7 @@ private void createMetadata(String recUuid, Element template) throws Exception, } addPrivileges(id, params.privileges, localGroups, dataMan, context, log); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); if (log.isDebugEnabled()) { log.debug(" - Commit " + id); diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java index dee3f502fd6..37a55048dd4 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geoPREST/Aligner.java @@ -227,7 +227,7 @@ private void addMetadata(RecordInfo ri) throws Exception { addPrivileges(id, params.getPrivileges(), localGroups, dataMan, context, log); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, Math.random() < 0.01, null); result.addedMetadata++; } @@ -272,7 +272,7 @@ private void updateMetadata(RecordInfo ri, String id) throws Exception { addCategories(metadata, params.getCategories(), localCateg, context, log, null, true); dataMan.flush(); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, Math.random() < 0.01, null); result.updatedMetadata++; } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java index 80b8187f4f6..c60ced142c1 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet/Aligner.java @@ -571,7 +571,7 @@ private String addMetadata(RecordInfo ri, Element md, Element info, boolean loca } context.getBean(MetadataRepository.class).save(metadata); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, Math.random() < 0.01, null); result.addedMetadata++; return id; @@ -834,7 +834,7 @@ private void updateMetadata(RecordInfo ri, String id, Element md, Element info, metadataRepository.save(metadata); // dataMan.flush(); - dataMan.indexMetadata(id, Math.random() < 0.01); + dataMan.indexMetadata(id, Math.random() < 0.01, null); } private void updateFile(String id, String file, String dir, String changeDate, diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java index 50bd384ccb7..1e6daddb0ee 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/geonet20/Aligner.java @@ -195,7 +195,7 @@ public HarvestResult align(Element result, String siteId) throws Exception { //--- maybe the metadata was unretrievable if (id != null) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); } } } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java index 74ba9d6cf0d..d23e185551f 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/localfilesystem/LocalFilesystemHarvester.java @@ -194,7 +194,7 @@ void updateMetadata(Element xml, final String id, GroupMapper localGroups, dataMan.flush(); if (indexAfterUpdate == true) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); } } @@ -244,7 +244,7 @@ String addMetadata(Element xml, String uuid, String schema, GroupMapper localGro dataMan.flush(); if (index) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); } return id; } diff --git a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java index 7c274a92a24..5126f6f8298 100644 --- a/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java +++ b/harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/oaipmh/Harvester.java @@ -389,7 +389,7 @@ private void addMetadata(XmlRequest t, RecordInfo ri, String processName, Maplocalhost http://${es.host}:${es.port} features + records diff --git a/schemas/dublin-core/src/main/plugin/dublin-core/index-fields/index.xsl b/schemas/dublin-core/src/main/plugin/dublin-core/index-fields/index.xsl new file mode 100644 index 00000000000..691de3a41b5 --- /dev/null +++ b/schemas/dublin-core/src/main/plugin/dublin-core/index-fields/index.xsl @@ -0,0 +1,203 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + metadata + dublin-core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/schemas/iso19139/src/main/plugin/iso19139/index-fields/fn.xsl b/schemas/iso19139/src/main/plugin/iso19139/index-fields/fn.xsl new file mode 100644 index 00000000000..92b94a655cc --- /dev/null +++ b/schemas/iso19139/src/main/plugin/iso19139/index-fields/fn.xsl @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/schemas/iso19139/src/main/plugin/iso19139/index-fields/index.xsl b/schemas/iso19139/src/main/plugin/iso19139/index-fields/index.xsl new file mode 100644 index 00000000000..3822793ca10 --- /dev/null +++ b/schemas/iso19139/src/main/plugin/iso19139/index-fields/index.xsl @@ -0,0 +1,919 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + metadata + iso19139 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dataset + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + POINT( + + ) + + + + {"type": "point", + "coordinates": "" + } + + + + + + POINT( + + ) + + + + {"type": "point", + "coordinates": "" + } + + + + + POLYGON(( + + , + + , + + , + + , + + )) + + + + + {"type": "polygon", + "coordinates": [ + + , + + , + + , + + , + + ]} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | + + | + + | + + + + + + view + + + download + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + { + org:"", + role:"" + } + + + diff --git a/schemas/iso19139/src/main/plugin/iso19139/index-fields/inspire-constant.xsl b/schemas/iso19139/src/main/plugin/iso19139/index-fields/inspire-constant.xsl new file mode 100644 index 00000000000..f2c623793c5 --- /dev/null +++ b/schemas/iso19139/src/main/plugin/iso19139/index-fields/inspire-constant.xsl @@ -0,0 +1,168 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + commission regulation (eu) + reglement (ue) + inspire + + + Commission Regulation (EU) No 1089/2010 of 23 November 2010 implementing Directive 2007/2/EC of the European Parliament and of the Council as regards interoperability of spatial data sets and services + Регламент (ЕС) № 1089/2010 на Комисията от 23 ноември 2010 година за прилагане на Директива 2007/2/ЕО на Европейския парламент и на Съвета по отношение на оперативната съвместимост на масиви от пространствени данни и услуги за пространствени данни + Reglamento (UE) n ° 1089/2010 de la Comisión, de 23 de noviembre de 2010 , por el que se aplica la Directiva 2007/2/CE del Parlamento Europeo y del Consejo en lo que se refiere a la interoperabilidad de los conjuntos y los servicios de datos espaciales + Nařízení Komise (EU) č. 1089/2010 ze dne 23. listopadu 2010 , kterým se provádí směrnice Evropského parlamentu a Rady 2007/2/ES, pokud jde o interoperabilitu sad prostorových dat a služeb prostorových dat + Kommissionens forordning (EU) nr. 1089/2010 af 23. november 2010 om gennemførelse af Europa-Parlamentets og Rådets direktiv 2007/2/EF for så vidt angår interoperabilitet for geodatasæt og -tjenester + Verordnung (EG) Nr. 1089/2010 der Kommission vom 23. November 2010 zur Durchführung der Richtlinie 2007/2/EG des Europäischen Parlaments und des Rates hinsichtlich der Interoperabilität von Geodatensätzen und -diensten + Komisjoni määrus (EL) nr 1089/2010, 23. november 2010 , millega rakendatakse Euroopa Parlamendi ja nõukogu direktiivi 2007/2/EÜ seoses ruumiandmekogumite ja -teenuste ristkasutatavusega + Κανονισμός (ΕΕ) αριθ. 1089/2010 της Επιτροπής, της 23ης Νοεμβρίου 2010 , σχετικά με την εφαρμογή της οδηγίας 2007/2/ΕΚ του Ευρωπαϊκού Κοινοβουλίου και του Συμβουλίου όσον αφορά τη διαλειτουργικότητα των συνόλων και των υπηρεσιών χωρικών δεδομένων + Règlement (UE) n ° 1089/2010 de la Commission du 23 novembre 2010 portant modalités d'application de la directive 2007/2/CE du Parlement européen et du Conseil en ce qui concerne l'interopérabilité des séries et des services de données géographiques +
Uredba Komisije (EU) br. 1089/2010 od 23. studenoga 2010. o provedbi Direktive 2007/2/EZ Europskog parlamenta i Vijeća o međuoperativnosti skupova prostornih podataka i usluga u vezi s prostornim podacima + Regolamento (UE) n. 1089/2010 della Commissione, del 23 novembre 2010 , recante attuazione della direttiva 2007/2/CE del Parlamento europeo e del Consiglio per quanto riguarda l'interoperabilità dei set di dati territoriali e dei servizi di dati territoriali + Komisijas Regula (ES) Nr. 1089/2010 ( 2010. gada 23. novembris ), ar kuru īsteno Eiropas Parlamenta un Padomes Direktīvu 2007/2/EK attiecībā uz telpisko datu kopu un telpisko datu pakalpojumu savstarpējo izmantojamību + 2010 m. lapkričio 23 d. Komisijos reglamentas (ES) Nr. 1089/2010, kuriuo įgyvendinamos Europos Parlamento ir Tarybos direktyvos 2007/2/EB nuostatos dėl erdvinių duomenų rinkinių ir paslaugų sąveikumo + A Bizottság 1089/2010/EU rendelete ( 2010. november 23. ) a 2007/2/EK európai parlamenti és tanácsi irányelv téradatkészletek és -szolgáltatások interoperabilitására vonatkozó rendelkezéseinek végrehajtásáról + Regolament tal-Kummissjoni (UE) Nru 1089/2010 tat- 23 ta' Novembru 2010 li jimplimenta d-Direttiva 2007/2/KE tal-Parlament Ewropew u tal-Kunsill fir-rigward tal-interoperabbiltà tas-settijiet ta’ dejta u servizzi ġeografiċi + Verordening (EU) nr. 1089/2010 van de Commissie van 23 november 2010 ter uitvoering van Richtlijn 2007/2/EG van het Europees Parlement en de Raad betreffende de interoperabiliteit van verzamelingen ruimtelijke gegevens en van diensten met betrekking tot ruimtelijke gegevens + Rozporządzenie Komisji (UE) nr 1089/2010 z dnia 23 listopada 2010 r. w sprawie wykonania dyrektywy 2007/2/WE Parlamentu Europejskiego i Rady w zakresie interoperacyjności zbiorów i usług danych przestrzennych + Regulamento (UE) n. ° 1089/2010 da Comissão, de 23 de Novembro de 2010 , que estabelece as disposições de execução da Directiva 2007/2/CE do Parlamento Europeu e do Conselho relativamente à interoperabilidade dos conjuntos e serviços de dados geográficos + Regulamentul (UE) nr. 1089/2010 al Comisiei din 23 noiembrie 2010 de punere în aplicare a Directivei 2007/2/CE a Parlamentului European și a Consiliului în ceea ce privește interoperabilitatea seturilor și serviciilor de date spațiale + Nariadenie Komisie (EÚ) č. 1089/2010 z 23. novembra 2010 , ktorým sa vykonáva smernica Európskeho parlamentu a Rady 2007/2/ES, pokiaľ ide o interoperabilitu súborov a služieb priestorových údajov + Uredba Komisije (EU) št. 1089/2010 z dne 23. novembra 2010 o izvajanju Direktive 2007/2/ES Evropskega parlamenta in Sveta glede medopravilnosti zbirk prostorskih podatkov in storitev v zvezi s prostorskimi podatki + Komission asetus (EU) N:o 1089/2010, annettu 23 päivänä marraskuuta 2010 , Euroopan parlamentin ja neuvoston direktiivin 2007/2/EY täytäntöönpanosta paikkatietoaineistojen ja -palvelujen yhteentoimivuuden osalta + Kommissionens förordning (EU) nr 1089/2010 av den 23 november 2010 om genomförande av Europaparlamentets och rådets direktiv 2007/2/EG vad gäller interoperabilitet för rumsliga datamängder och datatjänster + + + + + + + + + commission regulation (eu) + reglement (ue) + inspire + + + Commission Regulation (EC) No 976/2009 of 19 October 2009 implementing Directive 2007/2/EC of the European Parliament and of the Council as regards the Network Services + Регламент (ЕО) № 976/2009 на Комисията от 19 октомври 2009 година за прилагане на Директива 2007/2/ЕО на Европейския парламент и на Съвета по отношение на мрежовите услуги + Nařízení Komise (ES) č. 976/2009 ze dne 19. října 2009 , kterým se provádí směrnice Evropského parlamentu a Rady 2007/2/ES, pokud jde o síťové služby + Kommissionens forordning (EF) nr. 976/2009 af 19. oktober 2009 om gennemførelse af Europa-Parlamentets og Rådets direktiv 2007/2/EF for så vidt angår nettjenesterne + Verordnung (EG) Nr. 976/2009 der Kommission vom 19. Oktober 2009 zur Durchführung der Richtlinie 2007/2/EG des Europäischen Parlaments und des Rates hinsichtlich der Netzdienste + Komisjoni määrus (EÜ) nr 976/2009, 19. oktoober 2009 , millega rakendatakse Euroopa Parlamendi ja nõukogu direktiivi 2007/2/EÜ seoses võrguteenustega + Κανονισμός (ΕΚ) αριθ. 976/2009 της Επιτροπής, της 19ης Οκτωβρίου 2009 , για την υλοποίηση της οδηγίας 2007/2/ΕΚ του Ευρωπαϊκού Κοινοβουλίου και του Συμβουλίου όσον αφορά τις δικτυακές υπηρεσίες + Règlement (CE) n o 976/2009 de la Commission du 19 octobre 2009 portant modalités d’application de la directive 2007/2/CE du Parlement européen et du Conseil en ce qui concerne les services en réseau +
Uredba Komisije (EZ) br. 976/2009 od 19. listopada 2009. o provedbi Direktive 2007/2/EZ Europskog parlamenta i Vijeća u vezi s mrežnim uslugama + Regolamento (CE) n. 976/2009 della Commissione, del 19 ottobre 2009 , recante attuazione della direttiva 2007/2/CE del Parlamento europeo e del Consiglio per quanto riguarda i servizi di rete + Komisijas Regula (EK) Nr. 976/2009 ( 2009. gada 19. oktobris ), ar kuru īsteno Eiropas Parlamenta un Padomes Direktīvu 2007/2/EK attiecībā uz tīkla pakalpojumiem + 2009 m. spalio 19 d. Komisijos reglamentas (EB) Nr. 976/2009, kuriuo įgyvendinamos Europos Parlamento ir Tarybos direktyvos 2007/2/EB nuostatos dėl tinklo paslaugų + A Bizottság 976/2009/EK rendelete ( 2009. október 19. ) a 2007/2/EK európai parlamenti és tanácsi irányelv hálózati szolgáltatásokra vonatkozó rendelkezéseinek végrehajtásáról + Regolament tal-Kummissjoni (KE) Nru 976/2009 tad- 19 ta’ Ottubru 2009 li jimplimenta d-Direttiva 2007/2/KE tal-Parlament Ewropew u tal-Kunsill fir-rigward tas-Servizzi ta’ Netwerk + Verordening (EG) nr. 976/2009 van de Commissie van 19 oktober 2009 tot uitvoering van Richtlijn 2007/2/EG van het Europees Parlement en de Raad wat betreft de netwerkdiensten + Rozporządzenie Komisji (WE) nr 976/2009 z dnia 19 października 2009 r. w sprawie wykonania dyrektywy 2007/2/WE Parlamentu Europejskiego i Rady w zakresie usług sieciowych + Regulamento (CE) n. o 976/2009 da Comissão, de 19 de Outubro de 2009 , que estabelece as disposições de execução da Directiva 2007/2/CE do Parlamento Europeu e do Conselho no que respeita aos serviços de rede + Regulamentul (CE) nr. 976/2009 al Comisiei din 19 octombrie 2009 de aplicare a Directivei 2007/2/CE a Parlamentului European și a Consiliului în ceea ce privește serviciile de rețea + Nariadenie Komisie (ES) č. 976/2009 z 19. októbra 2009 , ktorým sa vykonáva smernica Európskeho parlamentu a Rady 2007/2/ES, pokiaľ ide o sieťové služby + Uredba Komisije (ES) št. 976/2009 z dne 19. oktobra 2009 o izvajanju Direktive 2007/2/ES Evropskega parlamenta in Sveta glede omrežnih storitev + Komission asetus (EY) N:o 976/2009, annettu 19 päivänä lokakuuta 2009 , Euroopan parlamentin ja neuvoston direktiivin 2007/2/EY täytäntöönpanosta verkkopalvelujen osalta + Kommissionens förordning (EG) nr 976/2009 av den 19 oktober 2009 om genomförande av Europaparlamentets och rådets direktiv 2007/2/EG med avseende på nättjänster + + + + diff --git a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java index f3427019f4a..9943467c222 100644 --- a/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java +++ b/services/src/main/java/org/fao/geonet/api/processing/XslProcessUtils.java @@ -153,7 +153,7 @@ public static Element process(ServiceContext context, String id, String process, // Always udpate metadata date stamp on metadata processing (minor edit has no effect). boolean updateDateStamp = true; dataMan.updateMetadata(context, id, processedMetadata, validate, ufo, index, language, new ISODate().toString(), updateDateStamp); - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); } report.addMetadataId(iId); diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java index fa7c4aea4fa..038b66a7db9 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataInsertDeleteApi.java @@ -974,7 +974,7 @@ public void apply(@Nonnull Metadata metadata) { }); } - dataMan.indexMetadata(id.get(0), true); + dataMan.indexMetadata(id.get(0), true, null); return Pair.read(Integer.valueOf(id.get(0)), uuid); } } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java index 19878c2b188..1fedef3f102 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java @@ -159,7 +159,7 @@ public void share( List privileges = sharing.getPrivileges(); setOperations(sharing, dataManager, context, metadata, operationMap, privileges); - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), true, null); } @@ -321,7 +321,7 @@ private boolean canPublishToAllGroup(ServiceContext context, DataManager dm, Met if (!hasValidation) { dm.doValidate(metadata.getDataInfo().getSchemaId(), metadata.getId() + "", new Document(metadata.getXmlData(false)), context.getLanguage()); - dm.indexMetadata(metadata.getId() + "", true); + dm.indexMetadata(metadata.getId() + "", true, null); } boolean isInvalid = @@ -481,7 +481,7 @@ public void setRecordGroup( metadata.getSourceInfo().setGroupOwner(groupIdentifier); metadataRepository.save(metadata); - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), true, null); } @@ -690,7 +690,7 @@ MetadataProcessingReport setRecordOwnership( report, dataManager, accessMan, metadataRepository, serviceContext, listOfUpdatedRecords, metadataUuid); dataManager.flush(); - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), true, null); } catch (Exception exception) { report.addError(exception); diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataTagApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataTagApi.java index 53349c73eba..dd118d2c1cb 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataTagApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataTagApi.java @@ -168,7 +168,7 @@ public void updateTags( } } - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), true, null); } @ApiOperation( @@ -216,7 +216,7 @@ public void deleteTags( } } - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), true, null); } diff --git a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java index 96378733633..99a61ee19f7 100644 --- a/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/MetadataWorkflowApi.java @@ -38,14 +38,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import java.util.HashSet; import java.util.Locale; @@ -145,6 +143,6 @@ public void status( //--- reindex metadata DataManager dataManager = appContext.getBean(DataManager.class); - dataManager.indexMetadata(String.valueOf(metadata.getId()), true); + dataManager.indexMetadata(String.valueOf(metadata.getId()), true, null); } } diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java index 12c29612936..9e27abb51f4 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/AjaxEditUtils.java @@ -594,7 +594,7 @@ public synchronized boolean addAttribute(String id, String ref, String name, Str dataManager.notifyMetadataChange(md, id); //--- update search criteria - dataManager.indexMetadata(id, true); + dataManager.indexMetadata(id, true, null); return true; } @@ -643,7 +643,7 @@ public synchronized boolean deleteAttribute(String id, String ref, String name, dataManager.notifyMetadataChange(md, id); //--- update search criteria - dataManager.indexMetadata(id, true); + dataManager.indexMetadata(id, true, null); return true; } diff --git a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java index e3180067cf9..18809ddc0d4 100644 --- a/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java +++ b/services/src/main/java/org/fao/geonet/api/records/editing/MetadataEditingApi.java @@ -44,7 +44,6 @@ import org.fao.geonet.kernel.metadata.StatusActions; import org.fao.geonet.kernel.metadata.StatusActionsFactory; import org.fao.geonet.kernel.setting.SettingManager; -import org.fao.geonet.repository.MetadataRepository; import org.fao.geonet.repository.MetadataValidationRepository; import org.fao.geonet.repository.OperationAllowedRepository; import org.fao.geonet.repository.specification.MetadataValidationSpecs; @@ -343,7 +342,7 @@ public Element saveEdits( } if (reindex) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); } ajaxEditUtils.removeMetadataEmbedded(session, id); diff --git a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java index 292f204bf6c..c42439193c4 100644 --- a/services/src/main/java/org/fao/geonet/api/site/SiteApi.java +++ b/services/src/main/java/org/fao/geonet/api/site/SiteApi.java @@ -34,6 +34,7 @@ import org.fao.geonet.GeonetContext; import org.fao.geonet.NodeInfo; import org.fao.geonet.SystemInfo; +import org.fao.geonet.Util; import org.fao.geonet.api.API; import org.fao.geonet.api.ApiParams; import org.fao.geonet.api.ApiUtils; @@ -44,6 +45,8 @@ import org.fao.geonet.exceptions.OperationAbortedEx; import org.fao.geonet.kernel.DataManager; import org.fao.geonet.kernel.GeonetworkDataDirectory; +import org.fao.geonet.kernel.search.EsSearchManager; +import org.fao.geonet.kernel.search.SearchManager; import org.fao.geonet.kernel.setting.SettingInfo; import org.fao.geonet.kernel.setting.SettingManager; import org.fao.geonet.kernel.setting.Settings; @@ -59,6 +62,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.http.HttpEntity; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -81,6 +85,7 @@ import static org.apache.commons.fileupload.util.Streams.checkFileName; import static org.fao.geonet.api.ApiParams.API_CLASS_CATALOG_TAG; +import static org.fao.geonet.api.ApiParams.API_PARAM_RECORD_UUIDS_OR_SELECTION; /** * @@ -484,6 +489,104 @@ public boolean isIndexing( return ApplicationContextHolder.get().getBean(DataManager.class).isIndexing(); } + @ApiOperation( + value = "Index", + notes = "", + nickname = "index") + @RequestMapping( + path = "/index", + produces = MediaType.APPLICATION_JSON_VALUE, + method = RequestMethod.PUT) + @PreAuthorize("hasRole('Editor')") + @ResponseBody + public HttpEntity index( + @ApiParam(value = "Drop and recreate index", + required = false) + @RequestParam(required = false, defaultValue = "true") + boolean reset, + @ApiParam(value = "Records having only XLinks", + required = false) + @RequestParam(required = false, defaultValue = "false") + boolean havingXlinkOnly, +// @ApiParam(value = API_PARAM_RECORD_UUIDS_OR_SELECTION, +// required = false, +// example = "") +// @RequestParam(required = false) +// String[] uuids, + @ApiParam( + value = ApiParams.API_PARAM_BUCKET_NAME, + required = false) + @RequestParam( + required = false + ) + String bucket, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + SearchManager searchMan = ApplicationContextHolder.get().getBean(SearchManager.class); + + searchMan.rebuildIndex(context, havingXlinkOnly, reset, bucket); + + return new HttpEntity<>(HttpStatus.CREATED); + } + + @ApiOperation( + value = "Index in Elastic", + notes = "", + nickname = "indexes") + @RequestMapping( + path = "/index/es", + produces = MediaType.APPLICATION_JSON_VALUE, + method = RequestMethod.PUT) + @PreAuthorize("hasRole('Editor')") + @ResponseBody + public HttpEntity indexEs( + @ApiParam(value = "Drop and recreate index", + required = false) + @RequestParam(required = false, defaultValue = "true") + boolean reset, + @ApiParam(value = "Records having only XLinks", + required = false) + @RequestParam(required = false, defaultValue = "false") + boolean havingXlinkOnly, +// @ApiParam(value = API_PARAM_RECORD_UUIDS_OR_SELECTION, +// required = false, +// example = "") +// @RequestParam(required = false) +// String[] uuids, + @ApiParam( + value = ApiParams.API_PARAM_BUCKET_NAME, + required = false) + @RequestParam( + required = false + ) + String bucket, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + EsSearchManager searchMan = ApplicationContextHolder.get().getBean(EsSearchManager.class); + + searchMan.rebuildIndex(context, havingXlinkOnly, reset, bucket); + + return new HttpEntity<>(HttpStatus.CREATED); + } + + @ApiOperation( + value = "Delete index in Elastic", + notes = "", + nickname = "deleteIndexes") + @RequestMapping( + path = "/index/es", + produces = MediaType.APPLICATION_JSON_VALUE, + method = RequestMethod.DELETE) + @PreAuthorize("hasRole('Editor')") + @ResponseBody + public HttpEntity deleteIndexEs() throws Exception { + EsSearchManager searchMan = ApplicationContextHolder.get().getBean(EsSearchManager.class); + searchMan.clearIndex(); + return new HttpEntity<>(HttpStatus.NO_CONTENT); + } + @ApiOperation( value = "Get build details", notes = "To know when and how this version of the application was built.", diff --git a/services/src/main/java/org/fao/geonet/services/metadata/BatchOpsMetadataReindexer.java b/services/src/main/java/org/fao/geonet/services/metadata/BatchOpsMetadataReindexer.java index 408ff64cbe4..c142f3ad0b6 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/BatchOpsMetadataReindexer.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/BatchOpsMetadataReindexer.java @@ -142,7 +142,7 @@ public static final class BatchOpsCallable implements Callable { public Void call() throws Exception { for (int i = beginIndex; i < beginIndex + count; i++) { boolean doIndex = beginIndex + count - 1 == i; - dm.indexMetadata(ids[i] + "", doIndex); + dm.indexMetadata(ids[i] + "", doIndex, null); } return null; diff --git a/services/src/main/java/org/fao/geonet/services/metadata/IndexRebuild.java b/services/src/main/java/org/fao/geonet/services/metadata/IndexRebuild.java index bf772080fdc..6b649f53777 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/IndexRebuild.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/IndexRebuild.java @@ -74,7 +74,7 @@ public Element exec(Element params, ServiceContext context) throws Exception { SearchManager searchMan = gc.getBean(SearchManager.class); - boolean info = searchMan.rebuildIndex(context, xlinks, reset, fromSelection); + boolean info = searchMan.rebuildIndex(context, xlinks, reset, fromSelection ? "metadata" : null); Element elResp = new Element(Jeeves.Elem.RESPONSE); elResp.addContent(new Element("status").setText((info ? "true" : "false"))); diff --git a/services/src/main/java/org/fao/geonet/services/metadata/Insert.java b/services/src/main/java/org/fao/geonet/services/metadata/Insert.java index 0b980840907..494dece56a6 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/Insert.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/Insert.java @@ -181,7 +181,7 @@ public void apply(@Nonnull Metadata metadata) { } // Index - dm.indexMetadata(id.get(0), true); + dm.indexMetadata(id.get(0), true, null); // Return response Element response = new Element(Jeeves.Elem.RESPONSE); diff --git a/services/src/main/java/org/fao/geonet/services/metadata/Publish.java b/services/src/main/java/org/fao/geonet/services/metadata/Publish.java index 9f6cfeef024..1b55cb65ecf 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/Publish.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/Publish.java @@ -175,7 +175,7 @@ private PublishReport exec(String commaSeparatedIds, boolean publish, boolean sk dataManager.doValidate(metadata.getDataInfo().getSchemaId(), metadata.getId() + "", new Document(metadata.getXmlData(false)), serviceContext.getLanguage()); - dataManager.indexMetadata(nextId, true); + dataManager.indexMetadata(nextId, true, null); } boolean isInvalid = diff --git a/services/src/main/java/org/fao/geonet/services/metadata/Update.java b/services/src/main/java/org/fao/geonet/services/metadata/Update.java index f16a56c55e9..c53353f1be3 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/Update.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/Update.java @@ -190,7 +190,7 @@ public Element serviceSpecificExec(Element params, ServiceContext context) throw } if (reindex) { - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); } diff --git a/services/src/main/java/org/fao/geonet/services/metadata/UpdateAdminOper.java b/services/src/main/java/org/fao/geonet/services/metadata/UpdateAdminOper.java index 012d5411f1c..185d44d5731 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/UpdateAdminOper.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/UpdateAdminOper.java @@ -49,7 +49,6 @@ import java.nio.file.Path; import java.util.List; -import java.util.StringTokenizer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -172,7 +171,7 @@ public Element serviceSpecificExec(Element params, ServiceContext context) throw } //--- index metadata - dm.indexMetadata(id, true); + dm.indexMetadata(id, true, null); //--- return id for showing return new Element(Jeeves.Elem.RESPONSE).addContent(new Element(Geonet.Elem.ID).setText(id)); @@ -199,7 +198,7 @@ private boolean canPublishToAllGroup(ServiceContext context, DataManager dm, int dm.doValidate(metadata.getDataInfo().getSchemaId(), mdId + "", new Document(metadata.getXmlData(false)), context.getLanguage()); - dm.indexMetadata(mdId + "", true); + dm.indexMetadata(mdId + "", true, null); } boolean isInvalid = diff --git a/services/src/main/java/org/fao/geonet/services/metadata/UpdateCategories.java b/services/src/main/java/org/fao/geonet/services/metadata/UpdateCategories.java index f1fb2744176..f987781cc85 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/UpdateCategories.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/UpdateCategories.java @@ -96,7 +96,7 @@ public void apply(@Nonnull Metadata entity) { } //--- index metadata - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); //--- return id for showing return new Element(Jeeves.Elem.RESPONSE).addContent(new Element(Geonet.Elem.ID).setText(id)); diff --git a/services/src/main/java/org/fao/geonet/services/metadata/UpdateGroupOwner.java b/services/src/main/java/org/fao/geonet/services/metadata/UpdateGroupOwner.java index 2159f0ed5d0..f4d68fc2898 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/UpdateGroupOwner.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/UpdateGroupOwner.java @@ -96,7 +96,7 @@ public Element serviceSpecificExec(Element params, ServiceContext context) throw metadataRepository.save(metadata); //--- index metadata - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); //--- return id for showing return new Element(Jeeves.Elem.RESPONSE). diff --git a/services/src/main/java/org/fao/geonet/services/metadata/UpdateStatus.java b/services/src/main/java/org/fao/geonet/services/metadata/UpdateStatus.java index 8f8ffb69367..dd212f040f3 100644 --- a/services/src/main/java/org/fao/geonet/services/metadata/UpdateStatus.java +++ b/services/src/main/java/org/fao/geonet/services/metadata/UpdateStatus.java @@ -102,7 +102,7 @@ public Element serviceSpecificExec(Element params, ServiceContext context) throw sa.statusChange(status, metadataIds, changeDate, changeMessage); //--- reindex metadata - dataMan.indexMetadata(id, true); + dataMan.indexMetadata(id, true, null); //--- return id for showing return new Element(Jeeves.Elem.RESPONSE).addContent(new Element(Geonet.Elem.ID).setText(id)); diff --git a/services/src/test/java/org/fao/geonet/services/metadata/PublishTest.java b/services/src/test/java/org/fao/geonet/services/metadata/PublishTest.java index c4f840c17a6..74b0992abb2 100644 --- a/services/src/test/java/org/fao/geonet/services/metadata/PublishTest.java +++ b/services/src/test/java/org/fao/geonet/services/metadata/PublishTest.java @@ -104,7 +104,7 @@ public void testPublishSingle() throws Exception { final String metadataId = metadataIds.get(0); allowedRepository.deleteAll(); - dataManager.indexMetadata(metadataId, true); + dataManager.indexMetadata(metadataId, true, null); MockHttpServletRequest request = new MockHttpServletRequest(); request.getSession(); @@ -117,7 +117,7 @@ public void testPublishSingle() throws Exception { assertPublishedInIndex(true, metadataId); allowedRepository.deleteAll(); - dataManager.indexMetadata(metadataId, true); + dataManager.indexMetadata(metadataId, true, null); assertPublishedInIndex(false, metadataId); SecurityContextHolder.clearContext(); diff --git a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js index fc20fb375c2..a65d5b0079b 100644 --- a/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js +++ b/web-ui/src/main/resources/catalog/js/admin/AdminToolsController.js @@ -444,6 +444,22 @@ type: 'danger'}); }); }; + $scope.indexInEs = function() { + return $http.put('../api/site/index/es') + .success(function(data) { + $rootScope.$broadcast('StatusUpdated', { + msg: $translate.instant('indexInEsDone'), + timeout: 2, + type: 'success'}); + }) + .error(function(data) { + $rootScope.$broadcast('StatusUpdated', { + title: $translate.instant('indexInEsDoneError'), + error: data, + timeout: 0, + type: 'danger'}); + }); + }; $scope.optimizeIndex = function() { return $http.get('admin.index.optimize') diff --git a/web-ui/src/main/resources/catalog/locales/en-admin.json b/web-ui/src/main/resources/catalog/locales/en-admin.json index b851e4c2c66..c190e0ce51c 100644 --- a/web-ui/src/main/resources/catalog/locales/en-admin.json +++ b/web-ui/src/main/resources/catalog/locales/en-admin.json @@ -459,6 +459,8 @@ "indexOptimizationInProgress": "Index optimization in progress ...", "indexingRecordsRelatedToTheThesaurus": "Records related to this thesaurus are being indexed ...", "information": "Information", + "indexInEs": "(beta) Index in remote index", + "indexInEsHelp": "Index the catalogue content in a remote Elasticsearch instance.", "inspireatom-errors": "Errors:", "inspireatom-harvestedfeeds": "Harvested feeds:", "inspireatom-loadAtomHarvesterReport": "Atom harvester report", diff --git a/web-ui/src/main/resources/catalog/templates/admin/tools/index.html b/web-ui/src/main/resources/catalog/templates/admin/tools/index.html index ec81dc62380..9d7380a3479 100644 --- a/web-ui/src/main/resources/catalog/templates/admin/tools/index.html +++ b/web-ui/src/main/resources/catalog/templates/admin/tools/index.html @@ -50,6 +50,21 @@

indexingInProgress

reloadLuceneConfigHelp
+
+ +
+
+
+ + +
+
indexInEsHelp
+
+

diff --git a/web/src/main/webResources/WEB-INF/config.properties b/web/src/main/webResources/WEB-INF/config.properties index 3f46f22bfd7..9cebd1c4176 100644 --- a/web/src/main/webResources/WEB-INF/config.properties +++ b/web/src/main/webResources/WEB-INF/config.properties @@ -13,5 +13,6 @@ usersavedselection.watchlist.recordurl=api/records/{{index:_uuid}} es.url=${es.url} es.index.features=${es.index.features} +es.index.records=${es.index.records} jms.url=${jms.url} diff --git a/workers/wfsfeature-harvester/pom.xml b/workers/wfsfeature-harvester/pom.xml index e52ea549ee2..aa205a1fea0 100644 --- a/workers/wfsfeature-harvester/pom.xml +++ b/workers/wfsfeature-harvester/pom.xml @@ -90,6 +90,12 @@ ${project.version} + + org.geonetwork-opensource + es-core + ${project.version} + + ${project.groupId} messaging @@ -107,22 +113,9 @@ spring-web - org.geotools gt-geojson - - - - io.searchbox - jest - diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/WFSHarvesterApi.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/WFSHarvesterApi.java index d7748c689d9..76cc3b491a0 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/WFSHarvesterApi.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/WFSHarvesterApi.java @@ -29,9 +29,10 @@ import org.apache.log4j.Logger; import org.fao.geonet.ApplicationContextHolder; import org.fao.geonet.api.API; +import org.fao.geonet.es.EsClient; import org.fao.geonet.harvester.wfsfeatures.event.WFSHarvesterEvent; import org.fao.geonet.harvester.wfsfeatures.model.WFSHarvesterParameter; -import org.fao.geonet.harvester.wfsfeatures.worker.EsClient; +import org.fao.geonet.harvester.wfsfeatures.worker.EsWFSFeatureIndexer; import org.fao.geonet.harvester.wfsfeatures.worker.WFSHarvesterRouteBuilder; import org.geonetwork.messaging.JMSMessager; import org.springframework.beans.factory.annotation.Autowired; @@ -43,7 +44,6 @@ import java.util.HashMap; -import static org.fao.geonet.harvester.wfsfeatures.worker.EsWFSFeatureIndexer.deleteFeatures; /** * Created by fgravin on 10/29/15. @@ -98,8 +98,10 @@ public JSONObject deleteWfs( @RequestParam String typeName) throws Exception { - - deleteFeatures(serviceUrl, typeName, Logger.getLogger(WFSHarvesterRouteBuilder.LOGGER_NAME), client); + EsWFSFeatureIndexer indexer = ApplicationContextHolder.get().getBean(EsWFSFeatureIndexer.class); + indexer.deleteFeatures(serviceUrl, typeName, + Logger.getLogger(WFSHarvesterRouteBuilder.LOGGER_NAME), + client); // TODO: Check user is authenticated ? JSONObject result = new JSONObject(); diff --git a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java index 22a7581c533..43735e54166 100644 --- a/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java +++ b/workers/wfsfeature-harvester/src/main/java/org/fao/geonet/harvester/wfsfeatures/worker/EsWFSFeatureIndexer.java @@ -37,6 +37,7 @@ import org.apache.camel.Exchange; import org.apache.jcs.access.exception.InvalidArgumentException; import org.apache.log4j.Logger; +import org.fao.geonet.es.EsClient; import org.fao.geonet.harvester.wfsfeatures.model.WFSHarvesterParameter; import org.geotools.data.FeatureSource; import org.geotools.data.Query; @@ -56,6 +57,7 @@ import org.opengis.metadata.extent.GeographicExtent; import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -64,7 +66,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.StringTokenizer; //import org.elasticsearch.common.xcontent.XContentBuilder; @@ -75,6 +76,9 @@ public class EsWFSFeatureIndexer { public static final String TREE_FIELD_SUFFIX = "_tree"; public static final String FEATURE_FIELD_PREFIX = "ft_"; + @Value("es.index.features") + private String index = "features"; + @Autowired private EsClient client; @@ -172,16 +176,16 @@ public void deleteFeatures(Exchange exchange) { deleteFeatures(url, typeName, logger, client); } - public static void deleteFeatures(String url, String typeName, Logger logger, + public void deleteFeatures(String url, String typeName, Logger logger, EsClient client) { logger.info(String.format( "Deleting features previously index from service '%s' and feature type '%s' in '%s'", - url, typeName, client.getCollection())); + url, typeName, index)); try { ZonedDateTime startTime = ZonedDateTime.now(); String msg = client.deleteByQuery( - client.getCollection(), + index, String.format( "+featureTypeId:\\\"%s#%s\\\"", url, typeName) ); @@ -191,7 +195,7 @@ public static void deleteFeatures(String url, String typeName, Logger logger, startTime = ZonedDateTime.now(); msg = client.deleteByQuery( - client.getCollection(), + index, String.format( "+id:\\\"%s#%s\\\"", url, typeName) ); @@ -202,18 +206,18 @@ public static void deleteFeatures(String url, String typeName, Logger logger, e.printStackTrace(); logger.error(String.format( "Error connecting to ES at '%s'. Error is %s.", - client.getCollection(), e.getMessage())); + index, e.getMessage())); } } private void saveHarvesterReport(WFSHarvesterExchangeState state) { Map report = state.getHarvesterReport(); - Index index = new Index.Builder(report) - .index(client.getCollection()) - .type(client.getCollection()) + Index search = new Index.Builder(report) + .index(index) + .type(index) .id(report.get("id").toString()).build(); try { - DocumentResult response = client.getClient().execute(index); + DocumentResult response = client.getClient().execute(search); logger.info(String.format( "Report saved for %s. Error is '%s'.", state.getParameters().getTypeName(), @@ -445,7 +449,7 @@ public void indexFeatures(Exchange exchange) throws Exception { typeName, nbOfFeatures)); } try { - client.bulkRequest(client.getCollection(), + client.bulkRequest(index, docCollection); } catch (Exception ex) { state.getHarvesterReport().put("error_ss", String.format( @@ -472,7 +476,7 @@ public void indexFeatures(Exchange exchange) throws Exception { typeName, nbOfFeatures)); } try { - client.bulkRequest(client.getCollection(), + client.bulkRequest(index, docCollection); } catch (Exception ex) { state.getHarvesterReport().put("error_ss", String.format( @@ -507,4 +511,8 @@ public void indexFeatures(Exchange exchange) throws Exception { saveHarvesterReport(state); } } + + public void setIndex(String index) { + this.index = index; + } } diff --git a/workers/wfsfeature-harvester/src/main/resources/config-spring-geonetwork.xml b/workers/wfsfeature-harvester/src/main/resources/config-spring-geonetwork.xml index 539bcba4079..3b05db48419 100644 --- a/workers/wfsfeature-harvester/src/main/resources/config-spring-geonetwork.xml +++ b/workers/wfsfeature-harvester/src/main/resources/config-spring-geonetwork.xml @@ -48,10 +48,7 @@ - - - + class="org.fao.geonet.es.EsClient"/>