Skip to content

Commit

Permalink
fix #2841 Enabled support for multiple Fess web applications on a sin…
Browse files Browse the repository at this point in the history 
…gle OpenSearch instance.
  • Loading branch information
@marevol
marevol committed Aug 22, 2024
1 parent ec63857 commit 327d2eb
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 10 deletions.
2 changes: 2 additions & 0 deletions src/main/java/org/codelibs/fess/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ public class Constants extends CoreLibConstants {

public static final String CRAWLING_USER_AGENT_SUFFIX = "; +http://fess.codelibs.org/bot.html)";

public static final String DOCUMENT_INDEX_SUFFIX_PATTERN = "yyyyMMddHHmmssSSS";

// fess properties
public static final String USER_INFO_PROPERTY = "user.info";

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ protected boolean startReindex(final boolean replaceAliases, final boolean reset
final String autoExpandReplicas) {
final String docIndex = "fess";
final String fromIndex = fessConfig.getIndexDocumentUpdateIndex();
final String toIndex = docIndex + "." + new SimpleDateFormat("yyyyMMddHHmm").format(new Date());
final String toIndex = docIndex + "." + new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date());
if (searchEngineClient.createIndex(docIndex, toIndex, numberOfShards, autoExpandReplicas, resetDictionaries)) {
searchEngineClient.admin().cluster().prepareHealth(toIndex).setWaitForYellowStatus().execute(ActionListener.wrap(response -> {
searchEngineClient.addMapping(docIndex, "doc", toIndex);
Expand Down
61 changes: 52 additions & 9 deletions src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,8 @@ public class SearchEngineClient implements Client {

private static final Logger logger = LogManager.getLogger(SearchEngineClient.class);

private static final String DOC_INDEX = "fess";

private static final String LOG_INDEX_PREFIX = "fess_log";

private static final String USER_INDEX_PREFIX = "fess_user";
Expand Down Expand Up @@ -268,6 +270,18 @@ public void open() {
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();

if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) {
String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
if (StringUtil.isBlank(dictionaryPath)) {
System.setProperty("fess.dictionary.path", fessConfig.getIndexDictionaryPrefix() + "/");
} else {
if (!dictionaryPath.endsWith("/")) {
dictionaryPath = dictionaryPath + "/";
}
System.setProperty("fess.dictionary.path", dictionaryPath + fessConfig.getIndexDictionaryPrefix() + "/");
}
}

String httpAddress = SystemUtil.getSearchEngineHttpAddress();
if (StringUtil.isBlank(httpAddress) && (runner == null)) {
switch (fessConfig.getFesenType()) {
Expand Down Expand Up @@ -322,7 +336,7 @@ public void open() {
final String configIndex = values[0];
final String configType = values[1];

final boolean isFessIndex = "fess".equals(configIndex);
final boolean isFessIndex = DOC_INDEX.equals(configIndex);
final String indexName;
if (isFessIndex) {
final boolean exists = existsIndex(fessConfig.getIndexDocumentUpdateIndex());
Expand All @@ -349,10 +363,10 @@ public void open() {
indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
} else if (configIndex.startsWith(USER_INDEX_PREFIX)) {
final String name = fessConfig.getIndexUserIndex();
indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
indexName = configIndex.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name);
} else if (configIndex.startsWith(LOG_INDEX_PREFIX)) {
final String name = fessConfig.getIndexLogIndex();
indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
indexName = configIndex.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name);
} else {
throw new FessSystemException("Unknown config index: " + configIndex);
}
Expand Down Expand Up @@ -553,7 +567,7 @@ public void addMapping(final String index, final String docType, final String in
final String mappingFile = getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + index + "/" + docType + ".json");
try {
source = FileUtil.readUTF8(mappingFile);
if ("fess".equals(index)) {
if (DOC_INDEX.equals(index)) {
for (final UnaryOperator<String> rule : docMappingRewriteRuleList) {
source = rule.apply(source);
}
Expand Down Expand Up @@ -624,7 +638,23 @@ protected void createAlias(final String index, final String createdIndexName) {
final File aliasConfigDir = ResourceUtil.getResourceAsFile(aliasConfigDirPath);
if (aliasConfigDir.isDirectory()) {
stream(aliasConfigDir.listFiles((dir, name) -> name.endsWith(".json"))).of(stream -> stream.forEach(f -> {
final String aliasName = f.getName().replaceFirst(".json$", "");
String aliasName = f.getName().replaceFirst(".json$", "");
if (index.equals(DOC_INDEX)) {
if ("fess.search".equals(aliasName)) {
aliasName = fessConfig.getIndexDocumentSearchIndex();
} else if ("fess.update".equals(aliasName)) {
aliasName = fessConfig.getIndexDocumentUpdateIndex();
}
} else if (index.startsWith(CONFIG_INDEX_PREFIX)) {
final String name = fessConfig.getIndexConfigIndex();
aliasName = aliasName.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
} else if (index.startsWith(USER_INDEX_PREFIX)) {
final String name = fessConfig.getIndexUserIndex();
aliasName = aliasName.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name);
} else if (index.startsWith(LOG_INDEX_PREFIX)) {
final String name = fessConfig.getIndexLogIndex();
aliasName = aliasName.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name);
}
String source = FileUtil.readUTF8(f);
if ("{}".equals(source.trim())) {
source = null;
Expand All @@ -646,13 +676,20 @@ protected void createAlias(final String index, final String createdIndexName) {
}

protected void sendConfigFiles(final String index) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
configListMap.getOrDefault(index, Collections.emptyList()).forEach(path -> {
String source = null;
final String filePath = indexConfigPath + "/" + index + "/" + path;
final String dictionaryPath;
if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) {
dictionaryPath = fessConfig.getIndexDictionaryPrefix() + "/" + path;
} else {
dictionaryPath = path;
}
try {
source = FileUtil.readUTF8(filePath);
try (CurlResponse response =
ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", path).body(source).execute()) {
ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictionaryPath).body(source).execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Register {} to {}", path, index);
} else if (response.getContentException() != null) {
Expand Down Expand Up @@ -703,14 +740,20 @@ public void flushConfigFiles(final Runnable callback) {
}

protected String generateNewIndexName(final String configIndex) {
return configIndex + "." + new SimpleDateFormat("yyyyMMdd").format(new Date());
return configIndex + "." + new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date());
}

protected void insertBulkData(final FessConfig fessConfig, final String configIndex, final String dataPath) {
try {
final BulkRequestBuilder builder = client.prepareBulk();
final ObjectMapper mapper = new ObjectMapper();
Arrays.stream(FileUtil.readUTF8(dataPath).split("\n")).reduce((prev, line) -> {
final String userIndex = fessConfig.getIndexUserIndex() + ".user";
Arrays.stream(FileUtil.readUTF8(dataPath).split("\n")).map(line -> {
return line//
.replace("\"_index\":\"fess_config.", "\"_index\":\"" + fessConfig.getIndexConfigIndex() + ".")//
.replace("\"_index\":\"fess_user.", "\"_index\":\"" + fessConfig.getIndexUserIndex() + ".")//
.replace("\"_index\":\"fess_log.", "\"_index\":\"" + fessConfig.getIndexLogIndex() + ".");
}).reduce((prev, line) -> {
try {
if (StringUtil.isBlank(prev)) {
final Map<String, Map<String, String>> result =
Expand All @@ -728,7 +771,7 @@ protected void insertBulkData(final FessConfig fessConfig, final String configIn
});
if (result.containsKey("index")) {
String source = line;
if ("fess_user.user".equals(configIndex)) {
if (userIndex.equals(configIndex)) {
source = source.replace("${fess.index.initial_password}", ComponentUtil.getComponent(FessLoginAssist.class)
.encryptPassword(fessConfig.getIndexUserInitialPassword()));
}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ public void init() {
settingsBuilder.indexTimeout(fessConfig.getIndexIndexTimeout());
settingsBuilder.indicesTimeout(fessConfig.getIndexIndicesTimeout());
settingsBuilder.searchTimeout(fessConfig.getIndexSearchTimeout());
settingsBuilder.setSettingsIndexName(fessConfig.getIndexDocumentSuggestIndex() + "_suggest");
suggester = Suggester.builder().settings(settingsBuilder).build(searchEngineClient, fessConfig.getIndexDocumentSuggestIndex());
if (ComponentUtil.hasPopularWordHelper()) {
popularWordHelper = ComponentUtil.getPopularWordHelper();
Expand Down
27 changes: 27 additions & 0 deletions src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -718,6 +718,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. fess_log */
String INDEX_LOG_INDEX = "index.log.index";

/** The key of the configuration. e.g. */
String INDEX_DICTIONARY_PREFIX = "index.dictionary.prefix";

/** The key of the configuration. e.g. lang,role,label,anchor,virtual_host */
String INDEX_ADMIN_ARRAY_FIELDS = "index.admin.array.fields";

Expand Down Expand Up @@ -4067,6 +4070,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getIndexLogIndex();

/**
* Get the value for the key 'index.dictionary.prefix'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexDictionaryPrefix();

/**
* Get the value for the key 'index.dictionary.prefix' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getIndexDictionaryPrefixAsInteger();

/**
* Get the value for the key 'index.admin.array.fields'. <br>
* The value is, e.g. lang,role,label,anchor,virtual_host <br>
Expand Down Expand Up @@ -8983,6 +9001,14 @@ public String getIndexLogIndex() {
return get(FessConfig.INDEX_LOG_INDEX);
}

public String getIndexDictionaryPrefix() {
return get(FessConfig.INDEX_DICTIONARY_PREFIX);
}

public Integer getIndexDictionaryPrefixAsInteger() {
return getAsInteger(FessConfig.INDEX_DICTIONARY_PREFIX);
}

public String getIndexAdminArrayFields() {
return get(FessConfig.INDEX_ADMIN_ARRAY_FIELDS);
}
Expand Down Expand Up @@ -11178,6 +11204,7 @@ protected java.util.Map<String, String> prepareGeneratedDefaultMap() {
defaultMap.put(FessConfig.INDEX_CONFIG_INDEX, "fess_config");
defaultMap.put(FessConfig.INDEX_USER_INDEX, "fess_user");
defaultMap.put(FessConfig.INDEX_LOG_INDEX, "fess_log");
defaultMap.put(FessConfig.INDEX_DICTIONARY_PREFIX, "");
defaultMap.put(FessConfig.INDEX_ADMIN_ARRAY_FIELDS, "lang,role,label,anchor,virtual_host");
defaultMap.put(FessConfig.INDEX_ADMIN_DATE_FIELDS, "expires,created,timestamp,last_modified");
defaultMap.put(FessConfig.INDEX_ADMIN_INTEGER_FIELDS, "");
Expand Down
1 change: 1 addition & 0 deletions src/main/resources/fess_config.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,7 @@ index.document.crawler.filter.number_of_replicas=1
index.config.index=fess_config
index.user.index=fess_user
index.log.index=fess_log
index.dictionary.prefix=

# doc management
index.admin.array.fields=lang,role,label,anchor,virtual_host
Expand Down

0 comments on commit 327d2eb

Please sign in to comment.