diff --git a/src/main/java/org/codelibs/fess/Constants.java b/src/main/java/org/codelibs/fess/Constants.java index c459248e6..bccfb9759 100644 --- a/src/main/java/org/codelibs/fess/Constants.java +++ b/src/main/java/org/codelibs/fess/Constants.java @@ -87,6 +87,8 @@ public class Constants extends CoreLibConstants { public static final String CRAWLING_USER_AGENT_SUFFIX = "; +http://fess.codelibs.org/bot.html)"; + public static final String DOCUMENT_INDEX_SUFFIX_PATTERN = "yyyyMMddHHmmssSSS"; + // fess properties public static final String USER_INFO_PROPERTY = "user.info"; diff --git a/src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java b/src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java index 60c36f809..0223edb73 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java @@ -300,7 +300,7 @@ protected boolean startReindex(final boolean replaceAliases, final boolean reset final String autoExpandReplicas) { final String docIndex = "fess"; final String fromIndex = fessConfig.getIndexDocumentUpdateIndex(); - final String toIndex = docIndex + "." + new SimpleDateFormat("yyyyMMddHHmm").format(new Date()); + final String toIndex = docIndex + "." + new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date()); if (searchEngineClient.createIndex(docIndex, toIndex, numberOfShards, autoExpandReplicas, resetDictionaries)) { searchEngineClient.admin().cluster().prepareHealth(toIndex).setWaitForYellowStatus().execute(ActionListener.wrap(response -> { searchEngineClient.addMapping(docIndex, "doc", toIndex); diff --git a/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java b/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java index ec9b1c75b..75ee350cd 100644 --- a/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java +++ b/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java @@ -184,6 +184,8 @@ public class SearchEngineClient implements Client { private static final Logger logger = LogManager.getLogger(SearchEngineClient.class); + private static final String DOC_INDEX = "fess"; + private static final String LOG_INDEX_PREFIX = "fess_log"; private static final String USER_INDEX_PREFIX = "fess_user"; @@ -268,6 +270,18 @@ public void open() { } final FessConfig fessConfig = ComponentUtil.getFessConfig(); + if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) { + String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY); + if (StringUtil.isBlank(dictionaryPath)) { + System.setProperty("fess.dictionary.path", fessConfig.getIndexDictionaryPrefix() + "/"); + } else { + if (!dictionaryPath.endsWith("/")) { + dictionaryPath = dictionaryPath + "/"; + } + System.setProperty("fess.dictionary.path", dictionaryPath + fessConfig.getIndexDictionaryPrefix() + "/"); + } + } + String httpAddress = SystemUtil.getSearchEngineHttpAddress(); if (StringUtil.isBlank(httpAddress) && (runner == null)) { switch (fessConfig.getFesenType()) { @@ -322,7 +336,7 @@ public void open() { final String configIndex = values[0]; final String configType = values[1]; - final boolean isFessIndex = "fess".equals(configIndex); + final boolean isFessIndex = DOC_INDEX.equals(configIndex); final String indexName; if (isFessIndex) { final boolean exists = existsIndex(fessConfig.getIndexDocumentUpdateIndex()); @@ -349,10 +363,10 @@ public void open() { indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name); } else if (configIndex.startsWith(USER_INDEX_PREFIX)) { final String name = fessConfig.getIndexUserIndex(); - indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name); + indexName = configIndex.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name); } else if (configIndex.startsWith(LOG_INDEX_PREFIX)) { final String name = fessConfig.getIndexLogIndex(); - indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name); + indexName = configIndex.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name); } else { throw new FessSystemException("Unknown config index: " + configIndex); } @@ -553,7 +567,7 @@ public void addMapping(final String index, final String docType, final String in final String mappingFile = getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + index + "/" + docType + ".json"); try { source = FileUtil.readUTF8(mappingFile); - if ("fess".equals(index)) { + if (DOC_INDEX.equals(index)) { for (final UnaryOperator rule : docMappingRewriteRuleList) { source = rule.apply(source); } @@ -624,7 +638,23 @@ protected void createAlias(final String index, final String createdIndexName) { final File aliasConfigDir = ResourceUtil.getResourceAsFile(aliasConfigDirPath); if (aliasConfigDir.isDirectory()) { stream(aliasConfigDir.listFiles((dir, name) -> name.endsWith(".json"))).of(stream -> stream.forEach(f -> { - final String aliasName = f.getName().replaceFirst(".json$", ""); + String aliasName = f.getName().replaceFirst(".json$", ""); + if (index.equals(DOC_INDEX)) { + if ("fess.search".equals(aliasName)) { + aliasName = fessConfig.getIndexDocumentSearchIndex(); + } else if ("fess.update".equals(aliasName)) { + aliasName = fessConfig.getIndexDocumentUpdateIndex(); + } + } else if (index.startsWith(CONFIG_INDEX_PREFIX)) { + final String name = fessConfig.getIndexConfigIndex(); + aliasName = aliasName.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name); + } else if (index.startsWith(USER_INDEX_PREFIX)) { + final String name = fessConfig.getIndexUserIndex(); + aliasName = aliasName.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name); + } else if (index.startsWith(LOG_INDEX_PREFIX)) { + final String name = fessConfig.getIndexLogIndex(); + aliasName = aliasName.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name); + } String source = FileUtil.readUTF8(f); if ("{}".equals(source.trim())) { source = null; @@ -646,13 +676,20 @@ protected void createAlias(final String index, final String createdIndexName) { } protected void sendConfigFiles(final String index) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); configListMap.getOrDefault(index, Collections.emptyList()).forEach(path -> { String source = null; final String filePath = indexConfigPath + "/" + index + "/" + path; + final String dictionaryPath; + if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) { + dictionaryPath = fessConfig.getIndexDictionaryPrefix() + "/" + path; + } else { + dictionaryPath = path; + } try { source = FileUtil.readUTF8(filePath); try (CurlResponse response = - ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", path).body(source).execute()) { + ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictionaryPath).body(source).execute()) { if (response.getHttpStatusCode() == 200) { logger.info("Register {} to {}", path, index); } else if (response.getContentException() != null) { @@ -703,14 +740,20 @@ public void flushConfigFiles(final Runnable callback) { } protected String generateNewIndexName(final String configIndex) { - return configIndex + "." + new SimpleDateFormat("yyyyMMdd").format(new Date()); + return configIndex + "." + new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date()); } protected void insertBulkData(final FessConfig fessConfig, final String configIndex, final String dataPath) { try { final BulkRequestBuilder builder = client.prepareBulk(); final ObjectMapper mapper = new ObjectMapper(); - Arrays.stream(FileUtil.readUTF8(dataPath).split("\n")).reduce((prev, line) -> { + final String userIndex = fessConfig.getIndexUserIndex() + ".user"; + Arrays.stream(FileUtil.readUTF8(dataPath).split("\n")).map(line -> { + return line// + .replace("\"_index\":\"fess_config.", "\"_index\":\"" + fessConfig.getIndexConfigIndex() + ".")// + .replace("\"_index\":\"fess_user.", "\"_index\":\"" + fessConfig.getIndexUserIndex() + ".")// + .replace("\"_index\":\"fess_log.", "\"_index\":\"" + fessConfig.getIndexLogIndex() + "."); + }).reduce((prev, line) -> { try { if (StringUtil.isBlank(prev)) { final Map> result = @@ -728,7 +771,7 @@ protected void insertBulkData(final FessConfig fessConfig, final String configIn }); if (result.containsKey("index")) { String source = line; - if ("fess_user.user".equals(configIndex)) { + if (userIndex.equals(configIndex)) { source = source.replace("${fess.index.initial_password}", ComponentUtil.getComponent(FessLoginAssist.class) .encryptPassword(fessConfig.getIndexUserInitialPassword())); } diff --git a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java index 945efcb3a..1d755f139 100644 --- a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java @@ -108,6 +108,7 @@ public void init() { settingsBuilder.indexTimeout(fessConfig.getIndexIndexTimeout()); settingsBuilder.indicesTimeout(fessConfig.getIndexIndicesTimeout()); settingsBuilder.searchTimeout(fessConfig.getIndexSearchTimeout()); + settingsBuilder.setSettingsIndexName(fessConfig.getIndexDocumentSuggestIndex() + "_suggest"); suggester = Suggester.builder().settings(settingsBuilder).build(searchEngineClient, fessConfig.getIndexDocumentSuggestIndex()); if (ComponentUtil.hasPopularWordHelper()) { popularWordHelper = ComponentUtil.getPopularWordHelper(); diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 7646ea337..8a8cb63d9 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -718,6 +718,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. fess_log */ String INDEX_LOG_INDEX = "index.log.index"; + /** The key of the configuration. e.g. */ + String INDEX_DICTIONARY_PREFIX = "index.dictionary.prefix"; + /** The key of the configuration. e.g. lang,role,label,anchor,virtual_host */ String INDEX_ADMIN_ARRAY_FIELDS = "index.admin.array.fields"; @@ -4067,6 +4070,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getIndexLogIndex(); + /** + * Get the value for the key 'index.dictionary.prefix'.
+ * The value is, e.g.
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getIndexDictionaryPrefix(); + + /** + * Get the value for the key 'index.dictionary.prefix' as {@link Integer}.
+ * The value is, e.g.
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getIndexDictionaryPrefixAsInteger(); + /** * Get the value for the key 'index.admin.array.fields'.
* The value is, e.g. lang,role,label,anchor,virtual_host
@@ -8983,6 +9001,14 @@ public String getIndexLogIndex() { return get(FessConfig.INDEX_LOG_INDEX); } + public String getIndexDictionaryPrefix() { + return get(FessConfig.INDEX_DICTIONARY_PREFIX); + } + + public Integer getIndexDictionaryPrefixAsInteger() { + return getAsInteger(FessConfig.INDEX_DICTIONARY_PREFIX); + } + public String getIndexAdminArrayFields() { return get(FessConfig.INDEX_ADMIN_ARRAY_FIELDS); } @@ -11178,6 +11204,7 @@ protected java.util.Map prepareGeneratedDefaultMap() { defaultMap.put(FessConfig.INDEX_CONFIG_INDEX, "fess_config"); defaultMap.put(FessConfig.INDEX_USER_INDEX, "fess_user"); defaultMap.put(FessConfig.INDEX_LOG_INDEX, "fess_log"); + defaultMap.put(FessConfig.INDEX_DICTIONARY_PREFIX, ""); defaultMap.put(FessConfig.INDEX_ADMIN_ARRAY_FIELDS, "lang,role,label,anchor,virtual_host"); defaultMap.put(FessConfig.INDEX_ADMIN_DATE_FIELDS, "expires,created,timestamp,last_modified"); defaultMap.put(FessConfig.INDEX_ADMIN_INTEGER_FIELDS, ""); diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index b537c01c2..d16dfe2cd 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -354,6 +354,7 @@ index.document.crawler.filter.number_of_replicas=1 index.config.index=fess_config index.user.index=fess_user index.log.index=fess_log +index.dictionary.prefix= # doc management index.admin.array.fields=lang,role,label,anchor,virtual_host