Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Indexing / Add the capability to index the catalogue content in Elasticsearch. #1966

Closed
wants to merge 4 commits into from
Closed

Indexing / Add the capability to index the catalogue content in Elasticsearch. #1966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
02eb774
ES / Add indexing all in remote index.
fxprunayre Apr 23, 2017
1584081
ES / Add site ref in document. Index a selection.
fxprunayre May 9, 2017
04826d0
Admin / Add indexing button. Add DC indexing.
fxprunayre May 31, 2017
2f63df7
Fix removal in Lucene.
fxprunayre May 31, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,11 @@
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-hibernate4</artifactId>
</dependency>
<dependency>
<groupId>org.geonetwork-opensource</groupId>
<artifactId>es-core</artifactId>
<version>3.3.0-SNAPSHOT</version>
</dependency>
</dependencies>

<build>
Expand Down
50 changes: 29 additions & 21 deletions core/src/main/java/org/fao/geonet/kernel/DataManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import org.fao.geonet.exceptions.ServiceNotAllowedEx;
import org.fao.geonet.exceptions.XSDValidationErrorEx;
import org.fao.geonet.kernel.schema.MetadataSchema;
import org.fao.geonet.kernel.search.ISearchManager;
import org.fao.geonet.kernel.search.SearchManager;
import org.fao.geonet.kernel.search.index.IndexingList;
import org.fao.geonet.kernel.setting.SettingManager;
Expand Down Expand Up @@ -415,7 +416,7 @@ public synchronized void init(ServiceContext context, Boolean force) throws Exce

// remove from index metadata not in DBMS
for (String id : docs.keySet()) {
getSearchManager().delete("_id", id);
getSearchManager().delete(id);

if (Log.isDebugEnabled(Geonet.DATA_MANAGER)) {
Log.debug(Geonet.DATA_MANAGER, "- removed record (" + id + ") from index");
Expand Down Expand Up @@ -451,6 +452,7 @@ public synchronized void rebuildIndexXLinkedMetadata(final ServiceContext contex
* Reindex all records in current selection.
*/
public synchronized void rebuildIndexForSelection(final ServiceContext context,
String bucket,
boolean clearXlink)
throws Exception {

Expand All @@ -459,8 +461,8 @@ public synchronized void rebuildIndexForSelection(final ServiceContext context,
UserSession session = context.getUserSession();
SelectionManager sm = SelectionManager.getManager(session);

synchronized (sm.getSelection("metadata")) {
for (Iterator<String> iter = sm.getSelection("metadata").iterator();
synchronized (sm.getSelection(bucket)) {
for (Iterator<String> iter = sm.getSelection(bucket).iterator();
iter.hasNext(); ) {
String uuid = (String) iter.next();
String id = getMetadataId(uuid);
Expand Down Expand Up @@ -557,7 +559,7 @@ public boolean isIndexing() {

public void indexMetadata(final List<String> metadataIds) throws Exception {
for (String metadataId : metadataIds) {
indexMetadata(metadataId, false);
indexMetadata(metadataId, false, null);
}

getSearchManager().forceIndexChanges();
Expand All @@ -566,7 +568,7 @@ public void indexMetadata(final List<String> metadataIds) throws Exception {
/**
* TODO javadoc.
*/
public void indexMetadata(final String metadataId, boolean forceRefreshReaders) throws Exception {
public void indexMetadata(final String metadataId, boolean forceRefreshReaders, ISearchManager searchManager) throws Exception {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think would be better to keep a method also with metadataId and forceRefreshReaders only that calls this one sending searchManager as null, that is used in all places changed to send searchManager to null. Wth the change there's quite changes where searchManager is send as null.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once we move totally to ES there will no need of this - having only one SearchManager. See what has been done last year in Solr POC https://github.com/titellus/core-geonetwork/blob/solr/core/src/main/java/org/fao/geonet/kernel/search/SolrSearchManager.java. I would say that we can tolerate this for the time being ?

indexLock.lock();
try {
if (waitForIndexing.contains(metadataId)) {
Expand Down Expand Up @@ -762,7 +764,12 @@ public void indexMetadata(final String metadataId, boolean forceRefreshReaders)
}
moreFields.add(SearchManager.makeField(Geonet.IndexFieldNames.VALID, isValid, true, true));
}
getSearchManager().index(getSchemaManager().getSchemaDir(schema), md, metadataId, moreFields, metadataType, root, forceRefreshReaders);

if (searchManager == null) {
getSearchManager().index(getSchemaManager().getSchemaDir(schema), md, metadataId, moreFields, metadataType, root, forceRefreshReaders);
} else {
searchManager.index(getSchemaManager().getSchemaDir(schema), md, metadataId, moreFields, metadataType, root, forceRefreshReaders);
}
} catch (Exception x) {
Log.error(Geonet.DATA_MANAGER, "The metadata document index with id=" + metadataId + " is corrupt/invalid - ignoring it. Error: " + x.getMessage(), x);
fullMd = null;
Expand Down Expand Up @@ -1283,7 +1290,7 @@ public String getNewVersion(String id) {
*/
public void setTemplate(final int id, final MetadataType type, final String title) throws Exception {
setTemplateExt(id, type);
indexMetadata(Integer.toString(id), true);
indexMetadata(Integer.toString(id), true, null);
}

/**
Expand All @@ -1304,7 +1311,7 @@ public void apply(@Nonnull Metadata metadata) {
*/
public void setHarvested(int id, String harvestUuid) throws Exception {
setHarvestedExt(id, harvestUuid);
indexMetadata(Integer.toString(id), true);
indexMetadata(Integer.toString(id), true, null);
}

/**
Expand Down Expand Up @@ -1436,7 +1443,7 @@ public void apply(Metadata entity) {
}
});

indexMetadata(Integer.toString(metadataId), true);
indexMetadata(Integer.toString(metadataId), true, null);

return rating;
}
Expand Down Expand Up @@ -1614,7 +1621,7 @@ public Metadata insertMetadata(ServiceContext context, Metadata newMetadata, Ele
copyDefaultPrivForGroup(context, stringId, groupId, fullRightsForGroup);

if (index) {
indexMetadata(stringId, forceRefreshReaders);
indexMetadata(stringId, forceRefreshReaders, null);
}

if (notifyChange) {
Expand Down Expand Up @@ -1745,12 +1752,13 @@ public boolean existsMetadataUuid(String uuid) throws Exception {
* Returns all the keywords in the system.
*/
public Element getKeywords() throws Exception {
Collection<String> keywords = getSearchManager().getTerms("keyword");
// TODO ES
// Collection<String> keywords = getSearchManager().getTerms("keyword");
Element el = new Element("keywords");

for (Object keyword : keywords) {
el.addContent(new Element("keyword").setText((String) keyword));
}
// for (Object keyword : keywords) {
// el.addContent(new Element("keyword").setText((String) keyword));
// }
return el;
}

Expand Down Expand Up @@ -1828,7 +1836,7 @@ public synchronized Metadata updateMetadata(final ServiceContext context, final
} finally {
if (index) {
//--- update search criteria
indexMetadata(metadataId, true);
indexMetadata(metadataId, true, null);
}
}
// Return an up to date metadata record
Expand Down Expand Up @@ -2138,7 +2146,7 @@ public synchronized void deleteMetadata(ServiceContext context, String metadataI
}

//--- update search criteria
getSearchManager().delete("_id", metadataId + "");
getSearchManager().delete(metadataId + "");
// _entityManager.flush();
// _entityManager.clear();
}
Expand All @@ -2152,7 +2160,7 @@ public synchronized void deleteMetadata(ServiceContext context, String metadataI
public synchronized void deleteMetadataGroup(ServiceContext context, String metadataId) throws Exception {
deleteMetadataFromDB(context, metadataId);
//--- update search criteria
getSearchManager().deleteGroup("_id", metadataId + "");
getSearchManager().delete(metadataId + "");
}

/**
Expand Down Expand Up @@ -2314,7 +2322,7 @@ private void transformMd(ServiceContext context, String metadataId, Element md,
notifyMetadataChange(md, metadataId);

//--- update search criteria
indexMetadata(metadataId, true);
indexMetadata(metadataId, true, null);
}
}

Expand Down Expand Up @@ -2630,7 +2638,7 @@ public String getCurrentStatus(int metadataId) throws Exception {
*/
public MetadataStatus setStatus(ServiceContext context, int id, int status, ISODate changeDate, String changeMessage) throws Exception {
MetadataStatus statusObject = setStatusExt(context, id, status, changeDate, changeMessage);
indexMetadata(Integer.toString(id), true);
indexMetadata(Integer.toString(id), true, null);
return statusObject;
}

Expand Down Expand Up @@ -3221,7 +3229,7 @@ public int batchDeleteMetadataAndUpdateIndex(Specification<Metadata> specificati
}

// Remove records from the index
getSearchManager().delete("_id", Lists.transform(idsOfMetadataToDelete, new Function<Integer, String>() {
getSearchManager().delete(Lists.transform(idsOfMetadataToDelete, new Function<Integer, String>() {
@Nullable
@Override
public String apply(@Nonnull Integer input) {
Expand All @@ -3247,7 +3255,7 @@ private <T> T getBean(Class<T> requiredType) {
return getApplicationContext().getBean(requiredType);
}

private SearchManager getSearchManager() {
private ISearchManager getSearchManager() {
return getBean(SearchManager.class);
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/fao/geonet/kernel/IndexMetadataTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public void run() {
}

try {
dataManager.indexMetadata(metadataId.toString(), false);
dataManager.indexMetadata(metadataId.toString(), false, null);
} catch (Exception e) {
Log.error(Geonet.INDEX_ENGINE, "Error indexing metadata '" + metadataId + "': " + e.getMessage()
+ "\n" + Util.getStackTrace(e));
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/fao/geonet/kernel/mef/Importer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ public void apply(@Nonnull final Metadata metadata) {
Files.createDirectories(priDir);


dm.indexMetadata(metadataIdMap.get(index), true);
dm.indexMetadata(metadataIdMap.get(index), true, null);
}

// --------------------------------------------------------------------
Expand Down
Loading