From 1e643bd49404315f390bd19054c9f8edff5dd32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Prunayre?= Date: Mon, 2 Sep 2024 10:48:01 +0200 Subject: [PATCH] Indexing / Draft field MUST not be an array (#8242) In case of XSL error, index document was containing an array instead of single value ``` "draft": [ "n", "n" ], ``` Draft field is added by the database info so no need to add it again on XSL error. Properly set it in case of indexing error. not causing issue in current app (but will be more strict in index document model). --- .../geonet/kernel/search/EsSearchManager.java | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java index 122e7e2930a..978ab63a750 100644 --- a/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java +++ b/core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java @@ -29,8 +29,8 @@ import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation; import co.elastic.clients.elasticsearch.core.search.Hit; -import co.elastic.clients.elasticsearch.indices.*; import co.elastic.clients.elasticsearch.indices.ExistsRequest; +import co.elastic.clients.elasticsearch.indices.*; import co.elastic.clients.transport.endpoints.BooleanResponse; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -73,8 +73,7 @@ import java.util.*; import static org.fao.geonet.constants.Geonet.IndexFieldNames.IS_TEMPLATE; -import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_FIELD; -import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_MSG; +import static org.fao.geonet.kernel.search.IndexFields.*; public class EsSearchManager implements ISearchManager { @@ -216,7 +215,6 @@ private void addMDFields(Element doc, Path schemaDir, doc.addContent(new Element(INDEXING_ERROR_FIELD).setText("true")); doc.addContent(createIndexingErrorMsgElement("indexingErrorMsg-indexingStyleSheetError", "error", Map.of("message", e.getMessage()))); - doc.addContent(new Element(IndexFields.DRAFT).setText("n")); } } @@ -225,7 +223,7 @@ private void addMoreFields(Element doc, Multimap fields) { fields.entries().forEach(e -> { Element newElement = new Element(e.getKey()) .setText(String.valueOf(e.getValue())); - if(objectFields.contains(e.getKey())) { + if (objectFields.contains(e.getKey())) { newElement.setAttribute("type", "object"); } doc.addContent(newElement); @@ -349,6 +347,7 @@ public BulkResponse updateFields(String id, Multimap fields, Set fields.asMap().forEach((e, v) -> fieldMap.put(e, v.toArray())); return updateFields(id, fieldMap, fieldsToRemove); } + public BulkResponse updateFields(String id, Map fieldMap, Set fieldsToRemove) throws IOException { fieldMap.put(Geonet.IndexFieldNames.INDEXING_DATE, new Date()); @@ -404,7 +403,7 @@ public void updateFieldsAsynch(String id, Map fields) { if (exception != null) { LOGGER.error("Failed to index {}", exception); } else { - LOGGER.info("Updated fields for document {}", id); + LOGGER.info("Updated fields for document {}", id); } }); } @@ -479,7 +478,7 @@ private void sendDocumentsToIndex() { } catch (Exception e) { LOGGER.error( "An error occurred while indexing {} documents in current indexing list. Error is {}.", - listOfDocumentsToIndex.size(), e.getMessage()); + listOfDocumentsToIndex.size(), e.getMessage()); } finally { // TODO: Trigger this async ? documents.keySet().forEach(uuid -> overviewFieldUpdater.process(uuid)); @@ -502,6 +501,7 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, String id = ""; String uuid = ""; String isTemplate = ""; + String isDraft = ""; String failureDoc = documents.get(e.id()); try { @@ -510,13 +510,14 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, id = node.get(IndexFields.DBID).asText(); uuid = node.get("uuid").asText(); isTemplate = node.get(IS_TEMPLATE).asText(); + isDraft = node.get(DRAFT).asText(); } catch (Exception ignoredException) { } docWithErrorInfo.put(IndexFields.DBID, id); docWithErrorInfo.put("uuid", uuid); docWithErrorInfo.put(IndexFields.RESOURCE_TITLE, resourceTitle); docWithErrorInfo.put(IS_TEMPLATE, isTemplate); - docWithErrorInfo.put(IndexFields.DRAFT, "n"); + docWithErrorInfo.put(IndexFields.DRAFT, isDraft); docWithErrorInfo.put(INDEXING_ERROR_FIELD, true); ArrayNode errors = docWithErrorInfo.putArray(INDEXING_ERROR_MSG); errors.add(createIndexingErrorMsgObject(e.error().reason(), "error", Map.of())); @@ -539,7 +540,7 @@ private void checkIndexResponse(BulkResponse bulkItemResponses, BulkResponse response = client.bulkRequest(defaultIndex, listErrorOfDocumentsToIndex); if (response.errors()) { LOGGER.error("Failed to save error documents {}.", - Arrays.toString(errorDocumentIds.toArray())); + Arrays.toString(errorDocumentIds.toArray())); } } } @@ -675,7 +676,7 @@ public ObjectNode documentToJson(Element xml) { mapper.readTree(node.getTextNormalize())); } catch (IOException e) { LOGGER.error("Parsing invalid JSON node {} for property {}. Error is: {}", - node.getTextNormalize(), propertyName, e.getMessage()); + node.getTextNormalize(), propertyName, e.getMessage()); } } else { arrayNode.add( @@ -694,7 +695,7 @@ public ObjectNode documentToJson(Element xml) { )); } catch (IOException e) { LOGGER.error("Parsing invalid JSON node {} for property {}. Error is: {}", - nodeElements.get(0).getTextNormalize(), propertyName, e.getMessage()); + nodeElements.get(0).getTextNormalize(), propertyName, e.getMessage()); } } else { doc.put(propertyName, @@ -707,7 +708,8 @@ public ObjectNode documentToJson(Element xml) { } - /** Field starting with _ not supported in Kibana + /** + * Field starting with _ not supported in Kibana * Those are usually GN internal fields */ private String getPropertyName(String name) { @@ -935,12 +937,12 @@ public boolean isIndexWritable(String indexName) throws IOException, Elasticsear String indexBlockRead = "index.blocks.read_only_allow_delete"; GetIndicesSettingsRequest request = GetIndicesSettingsRequest.of( - b -> b.index(indexName) - .name(indexBlockRead) + b -> b.index(indexName) + .name(indexBlockRead) ); GetIndicesSettingsResponse settings = this.client.getClient() - .indices().getSettings(request); + .indices().getSettings(request); IndexState indexState = settings.get(indexBlockRead); @@ -951,7 +953,7 @@ public boolean isIndexWritable(String indexName) throws IOException, Elasticsear /** * Make a JSON Object that properly represents an indexingErrorMsg, to be used in the index. * - * @param type either 'error' or 'warning' + * @param type either 'error' or 'warning' * @param string a string that is translatable (see, e.g., en-search.json) * @param values values that replace the placeholders in the `string` parameter * @return a json object that represents an indexingErrorMsg @@ -962,7 +964,7 @@ public ObjectNode createIndexingErrorMsgObject(String string, String type, Map valuesObject.put(k, String.valueOf(v))); + values.forEach((k, v) -> valuesObject.put(k, String.valueOf(v))); indexingErrorMsg.set("values", valuesObject); return indexingErrorMsg; } @@ -970,7 +972,7 @@ public ObjectNode createIndexingErrorMsgObject(String string, String type, Map