Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Indexing / Draft field MUST not be an array #8242

Merged
merged 1 commit into from
Sep 2, 2024
Merged

Indexing / Draft field MUST not be an array #8242

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 20 additions & 18 deletions core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.elasticsearch.indices.ExistsRequest;
import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.transport.endpoints.BooleanResponse;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
Expand Down Expand Up @@ -73,8 +73,7 @@
import java.util.*;

import static org.fao.geonet.constants.Geonet.IndexFieldNames.IS_TEMPLATE;
import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_FIELD;
import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_MSG;
import static org.fao.geonet.kernel.search.IndexFields.*;


public class EsSearchManager implements ISearchManager {
Expand Down Expand Up @@ -216,7 +215,6 @@ private void addMDFields(Element doc, Path schemaDir,
doc.addContent(new Element(INDEXING_ERROR_FIELD).setText("true"));
doc.addContent(createIndexingErrorMsgElement("indexingErrorMsg-indexingStyleSheetError", "error",
Map.of("message", e.getMessage())));
doc.addContent(new Element(IndexFields.DRAFT).setText("n"));
}
}

Expand All @@ -225,7 +223,7 @@ private void addMoreFields(Element doc, Multimap<String, Object> fields) {
fields.entries().forEach(e -> {
Element newElement = new Element(e.getKey())
.setText(String.valueOf(e.getValue()));
if(objectFields.contains(e.getKey())) {
if (objectFields.contains(e.getKey())) {
newElement.setAttribute("type", "object");
}
doc.addContent(newElement);
Expand Down Expand Up @@ -349,6 +347,7 @@ public BulkResponse updateFields(String id, Multimap<String, Object> fields, Set
fields.asMap().forEach((e, v) -> fieldMap.put(e, v.toArray()));
return updateFields(id, fieldMap, fieldsToRemove);
}

public BulkResponse updateFields(String id, Map<String, Object> fieldMap, Set<String> fieldsToRemove) throws IOException {
fieldMap.put(Geonet.IndexFieldNames.INDEXING_DATE, new Date());

Expand Down Expand Up @@ -404,7 +403,7 @@ public void updateFieldsAsynch(String id, Map<String, Object> fields) {
if (exception != null) {
LOGGER.error("Failed to index {}", exception);
} else {
LOGGER.info("Updated fields for document {}", id);
LOGGER.info("Updated fields for document {}", id);
}
});
}
Expand Down Expand Up @@ -479,7 +478,7 @@ private void sendDocumentsToIndex() {
} catch (Exception e) {
LOGGER.error(
"An error occurred while indexing {} documents in current indexing list. Error is {}.",
listOfDocumentsToIndex.size(), e.getMessage());
listOfDocumentsToIndex.size(), e.getMessage());
} finally {
// TODO: Trigger this async ?
documents.keySet().forEach(uuid -> overviewFieldUpdater.process(uuid));
Expand All @@ -502,6 +501,7 @@ private void checkIndexResponse(BulkResponse bulkItemResponses,
String id = "";
String uuid = "";
String isTemplate = "";
String isDraft = "";

String failureDoc = documents.get(e.id());
try {
Expand All @@ -510,13 +510,14 @@ private void checkIndexResponse(BulkResponse bulkItemResponses,
id = node.get(IndexFields.DBID).asText();
uuid = node.get("uuid").asText();
isTemplate = node.get(IS_TEMPLATE).asText();
isDraft = node.get(DRAFT).asText();
} catch (Exception ignoredException) {
}
docWithErrorInfo.put(IndexFields.DBID, id);
docWithErrorInfo.put("uuid", uuid);
docWithErrorInfo.put(IndexFields.RESOURCE_TITLE, resourceTitle);
docWithErrorInfo.put(IS_TEMPLATE, isTemplate);
docWithErrorInfo.put(IndexFields.DRAFT, "n");
docWithErrorInfo.put(IndexFields.DRAFT, isDraft);
docWithErrorInfo.put(INDEXING_ERROR_FIELD, true);
ArrayNode errors = docWithErrorInfo.putArray(INDEXING_ERROR_MSG);
errors.add(createIndexingErrorMsgObject(e.error().reason(), "error", Map.of()));
Expand All @@ -539,7 +540,7 @@ private void checkIndexResponse(BulkResponse bulkItemResponses,
BulkResponse response = client.bulkRequest(defaultIndex, listErrorOfDocumentsToIndex);
if (response.errors()) {
LOGGER.error("Failed to save error documents {}.",
Arrays.toString(errorDocumentIds.toArray()));
Arrays.toString(errorDocumentIds.toArray()));
}
}
}
Expand Down Expand Up @@ -674,7 +675,7 @@ public ObjectNode documentToJson(Element xml) {
mapper.readTree(node.getTextNormalize()));
} catch (IOException e) {
LOGGER.error("Parsing invalid JSON node {} for property {}. Error is: {}",
node.getTextNormalize(), propertyName, e.getMessage());
node.getTextNormalize(), propertyName, e.getMessage());
}
} else {
arrayNode.add(
Expand All @@ -693,7 +694,7 @@ public ObjectNode documentToJson(Element xml) {
));
} catch (IOException e) {
LOGGER.error("Parsing invalid JSON node {} for property {}. Error is: {}",
nodeElements.get(0).getTextNormalize(), propertyName, e.getMessage());
nodeElements.get(0).getTextNormalize(), propertyName, e.getMessage());
}
} else {
doc.put(propertyName,
Expand All @@ -706,7 +707,8 @@ public ObjectNode documentToJson(Element xml) {
}


/** Field starting with _ not supported in Kibana
/**
* Field starting with _ not supported in Kibana
* Those are usually GN internal fields
*/
private String getPropertyName(String name) {
Expand Down Expand Up @@ -934,12 +936,12 @@ public boolean isIndexWritable(String indexName) throws IOException, Elasticsear
String indexBlockRead = "index.blocks.read_only_allow_delete";

GetIndicesSettingsRequest request = GetIndicesSettingsRequest.of(
b -> b.index(indexName)
.name(indexBlockRead)
b -> b.index(indexName)
.name(indexBlockRead)
);

GetIndicesSettingsResponse settings = this.client.getClient()
.indices().getSettings(request);
.indices().getSettings(request);

IndexState indexState = settings.get(indexBlockRead);

Expand All @@ -950,7 +952,7 @@ public boolean isIndexWritable(String indexName) throws IOException, Elasticsear
/**
* Make a JSON Object that properly represents an indexingErrorMsg, to be used in the index.
*
* @param type either 'error' or 'warning'
* @param type either 'error' or 'warning'
* @param string a string that is translatable (see, e.g., en-search.json)
* @param values values that replace the placeholders in the `string` parameter
* @return a json object that represents an indexingErrorMsg
Expand All @@ -961,15 +963,15 @@ public ObjectNode createIndexingErrorMsgObject(String string, String type, Map<S
indexingErrorMsg.put("string", string);
indexingErrorMsg.put("type", type);
ObjectNode valuesObject = objectMapper.createObjectNode();
values.forEach((k,v) -> valuesObject.put(k, String.valueOf(v)));
values.forEach((k, v) -> valuesObject.put(k, String.valueOf(v)));
indexingErrorMsg.set("values", valuesObject);
return indexingErrorMsg;
}

/**
* Create an Element that represents an indexingErrorMsg object, to be used in the index.
*
* @param type either 'error' or 'warning'
* @param type either 'error' or 'warning'
* @param string a string that is translatable (see, e.g., en-search.json)
* @param values values that replace the placeholders in the `string` parameter
* @return an Element that represents an indexingErrorMsg
Expand Down
Loading