Skip to content

Commit

Permalink
Indexing / Draft field MUST not be an array (#8242)
Browse files Browse the repository at this point in the history 
In case of XSL error, index document was containing an array instead of single value

```
"draft": [
      "n",
      "n"
    ],
```

Draft field is added by the database info so no need to add it again on XSL error. Properly set it in case of indexing error.

not causing issue in current app (but will be more strict in index document model).
  • Loading branch information
@fxprunayre
fxprunayre committed Sep 2, 2024
1 parent 10c99f6 commit 1e643bd
Showing 1 changed file with 20 additions and 18 deletions.
38 changes: 20 additions & 18 deletions core/src/main/java/org/fao/geonet/kernel/search/EsSearchManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.elasticsearch.indices.ExistsRequest;
import co.elastic.clients.elasticsearch.indices.*;
import co.elastic.clients.transport.endpoints.BooleanResponse;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
Expand Down Expand Up @@ -73,8 +73,7 @@
import java.util.*;

import static org.fao.geonet.constants.Geonet.IndexFieldNames.IS_TEMPLATE;
import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_FIELD;
import static org.fao.geonet.kernel.search.IndexFields.INDEXING_ERROR_MSG;
import static org.fao.geonet.kernel.search.IndexFields.*;


public class EsSearchManager implements ISearchManager {
Expand Down Expand Up @@ -216,7 +215,6 @@ private void addMDFields(Element doc, Path schemaDir,
doc.addContent(new Element(INDEXING_ERROR_FIELD).setText("true"));
doc.addContent(createIndexingErrorMsgElement("indexingErrorMsg-indexingStyleSheetError", "error",
Map.of("message", e.getMessage())));
doc.addContent(new Element(IndexFields.DRAFT).setText("n"));
}
}

Expand All @@ -225,7 +223,7 @@ private void addMoreFields(Element doc, Multimap<String, Object> fields) {
fields.entries().forEach(e -> {
Element newElement = new Element(e.getKey())
.setText(String.valueOf(e.getValue()));
if(objectFields.contains(e.getKey())) {
if (objectFields.contains(e.getKey())) {
newElement.setAttribute("type", "object");
}
doc.addContent(newElement);
Expand Down Expand Up @@ -349,6 +347,7 @@ public BulkResponse updateFields(String id, Multimap<String, Object> fields, Set
fields.asMap().forEach((e, v) -> fieldMap.put(e, v.toArray()));
return updateFields(id, fieldMap, fieldsToRemove);
}

public BulkResponse updateFields(String id, Map<String, Object> fieldMap, Set<String> fieldsToRemove) throws IOException {
fieldMap.put(Geonet.IndexFieldNames.INDEXING_DATE, new Date());

Expand Down Expand Up @@ -404,7 +403,7 @@ public void updateFieldsAsynch(String id, Map<String, Object> fields) {
if (exception != null) {
LOGGER.error("Failed to index {}", exception);
} else {
LOGGER.info("Updated fields for document {}", id);
LOGGER.info("Updated fields for document {}", id);
}
});
}
Expand Down Expand Up @@ -479,7 +478,7 @@ private void sendDocumentsToIndex() {
} catch (Exception e) {
LOGGER.error(
"An error occurred while indexing {} documents in current indexing list. Error is {}.",
listOfDocumentsToIndex.size(), e.getMessage());
listOfDocumentsToIndex.size(), e.getMessage());
} finally {
// TODO: Trigger this async ?
documents.keySet().forEach(uuid -> overviewFieldUpdater.process(uuid));
Expand All @@ -502,6 +501,7 @@ private void checkIndexResponse(BulkResponse bulkItemResponses,
String id = "";
String uuid = "";
String isTemplate = "";
String isDraft = "";

String failureDoc = documents.get(e.id());
try {
Expand All @@ -510,13 +510,14 @@ private void checkIndexResponse(BulkResponse bulkItemResponses,
id = node.get(IndexFields.DBID).asText();
uuid = node.get("uuid").asText();
isTemplate = node.get(IS_TEMPLATE).asText();
isDraft = node.get(DRAFT).asText();
} catch (Exception ignoredException) {
}
docWithErrorInfo.put(IndexFields.DBID, id);
docWithErrorInfo.put("uuid", uuid);
docWithErrorInfo.put(IndexFields.RESOURCE_TITLE, resourceTitle);
docWithErrorInfo.put(IS_TEMPLATE, isTemplate);
docWithErrorInfo.put(IndexFields.DRAFT, "n");
docWithErrorInfo.put(IndexFields.DRAFT, isDraft);
docWithErrorInfo.put(INDEXING_ERROR_FIELD, true);
ArrayNode errors = docWithErrorInfo.putArray(INDEXING_ERROR_MSG);
errors.add(createIndexingErrorMsgObject(e.error().reason(), "error", Map.of()));
Expand All @@ -539,7 +540,7 @@ private void checkIndexResponse(BulkResponse bulkItemResponses,
BulkResponse response = client.bulkRequest(defaultIndex, listErrorOfDocumentsToIndex);
if (response.errors()) {
LOGGER.error("Failed to save error documents {}.",
Arrays.toString(errorDocumentIds.toArray()));
Arrays.toString(errorDocumentIds.toArray()));
}
}
}
Expand Down Expand Up @@ -675,7 +676,7 @@ public ObjectNode documentToJson(Element xml) {
mapper.readTree(node.getTextNormalize()));
} catch (IOException e) {
LOGGER.error("Parsing invalid JSON node {} for property {}. Error is: {}",
node.getTextNormalize(), propertyName, e.getMessage());
node.getTextNormalize(), propertyName, e.getMessage());
}
} else {
arrayNode.add(
Expand All @@ -694,7 +695,7 @@ public ObjectNode documentToJson(Element xml) {
));
} catch (IOException e) {
LOGGER.error("Parsing invalid JSON node {} for property {}. Error is: {}",
nodeElements.get(0).getTextNormalize(), propertyName, e.getMessage());
nodeElements.get(0).getTextNormalize(), propertyName, e.getMessage());
}
} else {
doc.put(propertyName,
Expand All @@ -707,7 +708,8 @@ public ObjectNode documentToJson(Element xml) {
}


/** Field starting with _ not supported in Kibana
/**
* Field starting with _ not supported in Kibana
* Those are usually GN internal fields
*/
private String getPropertyName(String name) {
Expand Down Expand Up @@ -935,12 +937,12 @@ public boolean isIndexWritable(String indexName) throws IOException, Elasticsear
String indexBlockRead = "index.blocks.read_only_allow_delete";

GetIndicesSettingsRequest request = GetIndicesSettingsRequest.of(
b -> b.index(indexName)
.name(indexBlockRead)
b -> b.index(indexName)
.name(indexBlockRead)
);

GetIndicesSettingsResponse settings = this.client.getClient()
.indices().getSettings(request);
.indices().getSettings(request);

IndexState indexState = settings.get(indexBlockRead);

Expand All @@ -951,7 +953,7 @@ public boolean isIndexWritable(String indexName) throws IOException, Elasticsear
/**
* Make a JSON Object that properly represents an indexingErrorMsg, to be used in the index.
*
* @param type either 'error' or 'warning'
* @param type either 'error' or 'warning'
* @param string a string that is translatable (see, e.g., en-search.json)
* @param values values that replace the placeholders in the `string` parameter
* @return a json object that represents an indexingErrorMsg
Expand All @@ -962,15 +964,15 @@ public ObjectNode createIndexingErrorMsgObject(String string, String type, Map<S
indexingErrorMsg.put("string", string);
indexingErrorMsg.put("type", type);
ObjectNode valuesObject = objectMapper.createObjectNode();
values.forEach((k,v) -> valuesObject.put(k, String.valueOf(v)));
values.forEach((k, v) -> valuesObject.put(k, String.valueOf(v)));
indexingErrorMsg.set("values", valuesObject);
return indexingErrorMsg;
}

/**
* Create an Element that represents an indexingErrorMsg object, to be used in the index.
*
* @param type either 'error' or 'warning'
* @param type either 'error' or 'warning'
* @param string a string that is translatable (see, e.g., en-search.json)
* @param values values that replace the placeholders in the `string` parameter
* @return an Element that represents an indexingErrorMsg
Expand Down

0 comments on commit 1e643bd

Please sign in to comment.