From 4e81c6c3ed1c68c08fe3c6cfb48a781c65a3a7e2 Mon Sep 17 00:00:00 2001 From: cyjseagull Date: Tue, 27 Aug 2024 20:51:49 +0800 Subject: [PATCH] add task-plugin and spi --- build.gradle | 3 + db/wedpr_ddl.sql | 17 +- settings.gradle | 9 + wedpr-components/spi/build.gradle | 15 ++ .../wedpr/components/spi/plugin/SPIInfo.java | 32 +++ .../components/spi/plugin/SPILoader.java | 75 ++++++ .../components/spi/plugin/SPIObject.java | 38 +++ wedpr-components/task-plugin/api/build.gradle | 16 ++ .../task/plugin/api/CommandExecutor.java | 108 +++++++++ .../task/plugin/api/TaskBuilder.java | 19 ++ .../task/plugin/api/TaskBuilderFactory.java | 24 ++ .../task/plugin/api/TaskExecutionContext.java | 46 ++++ .../task/plugin/api/TaskInterface.java | 27 +++ .../task/plugin/api/TaskResponse.java | 32 +++ .../task/plugin/api/WorkerExecutor.java | 22 ++ .../plugin/api/model/CommandTaskConfig.java | 79 +++++++ .../model/CommandTaskExecutionContext.java | 58 +++++ .../plugin/api/model/CommandTaskResponse.java | 33 +++ .../plugin/api/parameters/Parameters.java | 21 ++ .../api/shell/BaseShellBuilderImpl.java | 220 ++++++++++++++++++ .../plugin/api/shell/BaseShellLauncher.java | 40 ++++ .../task/plugin/api/shell/ShellBuilder.java | 34 +++ .../plugin/api/shell/ShellBuilderFactory.java | 46 ++++ .../api/shell/ShellCommandExecutor.java | 24 ++ .../task/plugin/api/shell/ShellConstant.java | 30 +++ .../task/plugin/api/shell/ShellLauncher.java | 21 ++ .../task/plugin/api/shell/ShellType.java | 44 ++++ .../plugin/api/shell/bash/BashBuilder.java | 50 ++++ .../plugin/api/shell/bash/BashLauncher.java | 24 ++ .../task/plugin/loader/TaskPluginLoader.java | 66 ++++++ .../task-plugin/shell/build.gradle | 15 ++ .../task/plugin/shell/ShellParameters.java | 42 ++++ .../task/plugin/shell/ShellTask.java | 58 +++++ .../task/plugin/shell/ShellTaskBuilder.java | 26 +++ .../plugin/shell/ShellTaskBuilderFactory.java | 34 +++ .../com/webank/wedpr/core/utils/Common.java | 27 +++ .../com/webank/wedpr/core/utils/Constant.java | 5 + .../webank/wedpr/core/utils/FileUtils.java | 31 +++ .../wedpr/core/utils/PropertiesHelper.java | 8 + 39 files changed, 1518 insertions(+), 1 deletion(-) create mode 100644 wedpr-components/spi/build.gradle create mode 100644 wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIInfo.java create mode 100644 wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPILoader.java create mode 100644 wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIObject.java create mode 100644 wedpr-components/task-plugin/api/build.gradle create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/CommandExecutor.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilder.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilderFactory.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskExecutionContext.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskInterface.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskResponse.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/WorkerExecutor.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskConfig.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskExecutionContext.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskResponse.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/parameters/Parameters.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellBuilderImpl.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellLauncher.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilder.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilderFactory.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellCommandExecutor.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellConstant.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellLauncher.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellType.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashBuilder.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashLauncher.java create mode 100644 wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/loader/TaskPluginLoader.java create mode 100644 wedpr-components/task-plugin/shell/build.gradle create mode 100644 wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellParameters.java create mode 100644 wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTask.java create mode 100644 wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilder.java create mode 100644 wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilderFactory.java create mode 100644 wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/FileUtils.java diff --git a/build.gradle b/build.gradle index 4b882600..c83b776c 100644 --- a/build.gradle +++ b/build.gradle @@ -45,6 +45,7 @@ ext { springfoxVersion = "3.0.0" guavaVersion = "32.0.1-jre" bcprovVersion = "1.78.1" + googleAutoServiceVersion = "1.1.1" } // check.dependsOn integrationTest @@ -135,6 +136,7 @@ allprojects { implementation logger, spring, spring_boot implementation("org.projectlombok:lombok:${lombokVersion}") + implementation("com.google.auto.service:auto-service:${googleAutoServiceVersion}") implementation("org.apache.commons:commons-lang3:${apacheCommonLangVersion}") implementation("io.springfox:springfox-boot-starter:${springfoxVersion}") implementation("com.google.guava:guava:${guavaVersion}") @@ -144,6 +146,7 @@ allprojects { testImplementation ("junit:junit:${junitVersion}") annotationProcessor("org.projectlombok:lombok:${lombokVersion}") + annotationProcessor("com.google.auto.service:auto-service:${googleAutoServiceVersion}") } clean.doLast { diff --git a/db/wedpr_ddl.sql b/db/wedpr_ddl.sql index 93de65f0..89f50fad 100644 --- a/db/wedpr_ddl.sql +++ b/db/wedpr_ddl.sql @@ -279,4 +279,19 @@ CREATE TABLE IF NOT EXISTS wedpr_permission ( create_by VARCHAR(20) NOT NULL DEFAULT '', update_by VARCHAR(20) NOT NULL DEFAULT '', PRIMARY KEY (permission_id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC; \ No newline at end of file + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC; + +-- the jupyter table +create table if not exists `wedpr_jupyter_table`( + `id` varchar(64) not null comment "Jupyter资源的ID", + `owner` varchar(255) not null comment "Jupyter属主", + `agency` varchar(255) not null comment "Jupyter所属机构", + `access_entrypoint` text comment "Jupyter访问入口", + `setting` longtext comment "Jupyter配置", + `status` int comment "Jupyter状态", + `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP comment "创建时间", + `last_update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP comment "更新时间", + PRIMARY KEY (id), + index owner_index(`owner`(128)), + index status_index(`status`) +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC; diff --git a/settings.gradle b/settings.gradle index ee7e0789..28314ac9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -59,6 +59,15 @@ project(":wedpr-components-scheduler").projectDir=file("wedpr-components/schedul include "wedpr-components-security" project(":wedpr-components-security").projectDir=file("wedpr-components/security") +include ":wedpr-components-task-plugin-api" +project(":wedpr-components-task-plugin-api").projectDir=file("wedpr-components/task-plugin/api") + +include ":wedpr-components-task-plugin-shell" +project(":wedpr-components-task-plugin-shell").projectDir=file("wedpr-components/task-plugin/shell") + +include ":wedpr-components-spi" +project(":wedpr-components-spi").projectDir=file("wedpr-components/spi") + include "wedpr-components-user" project(":wedpr-components-user").projectDir=file("wedpr-components/user") diff --git a/wedpr-components/spi/build.gradle b/wedpr-components/spi/build.gradle new file mode 100644 index 00000000..2222d9d7 --- /dev/null +++ b/wedpr-components/spi/build.gradle @@ -0,0 +1,15 @@ +// Apply the java-library plugin to add support for Java Library +plugins { + id 'java' + id 'com.github.sherter.google-java-format' +} +dependencies{ + compile project(":wedpr-core-utils") +} +googleJavaFormat { + //toolVersion = '1.7' + options style: 'AOSP' + source = sourceSets*.allJava + include '**/*.java' + //source = *.allJava +} diff --git a/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIInfo.java b/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIInfo.java new file mode 100644 index 00000000..1373e5c5 --- /dev/null +++ b/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIInfo.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.spi.plugin; + +import lombok.Data; + +@Data +public class SPIInfo { + private String name; + private Integer priority = 0; + + public SPIInfo(String name) { + this.name = name; + } + + public SPIInfo(String name, Integer priority) { + this.name = name; + this.priority = priority; + } +} diff --git a/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPILoader.java b/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPILoader.java new file mode 100644 index 00000000..d4a4345d --- /dev/null +++ b/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPILoader.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.spi.plugin; + +import com.webank.wedpr.core.utils.WeDPRException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; +import lombok.SneakyThrows; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SPILoader { + private static final Logger logger = LoggerFactory.getLogger(SPILoader.class); + private final Map spiObjectMap = new HashMap<>(); + + @SneakyThrows(Exception.class) + public SPILoader(Class spiClass) { + for (T spiObject : ServiceLoader.load(spiClass)) { + // load new spi object + if (!spiObjectMap.containsKey(spiObject.getSpiInfo().getName())) { + spiObjectMap.put(spiObject.getSpiInfo().getName(), spiObject); + continue; + } + // spi conflict, load the higher priority object + T existedSPIObject = spiObjectMap.get(spiObject.getSpiInfo().getName()); + if (existedSPIObject.equals(spiObject)) { + String errorMsg = + String.format( + "load spi failed for conflict, there are two spi objects with same name '%s' and priority '%s'", + spiObject.getSpiInfo().getName(), + spiObject.getSpiInfo().getPriority()); + logger.error(errorMsg); + throw new WeDPRException(errorMsg); + } + if (existedSPIObject.compareTo(spiObject) > 0) { + logger.info( + "load new spi object with higher priority, name: {}, oldPriority: {}, newPriority: {}", + spiObject.getSpiInfo().getName(), + existedSPIObject.getSpiInfo().getPriority(), + spiObject.getSpiInfo().getPriority()); + spiObjectMap.put(spiObject.getSpiInfo().getName(), spiObject); + continue; + } + logger.info( + "Ignore spi object with lower priority, name: {}, priority: {}, currentPriority: {}", + spiObject.getSpiInfo().getName(), + spiObject.getSpiInfo().getPriority(), + existedSPIObject.getSpiInfo().getPriority()); + } + } + + public Map getSpiObjectMap() { + return spiObjectMap; + } + + public T getSPIObjectByName(String spiName) { + if (!spiObjectMap.containsKey(spiName)) { + return null; + } + return spiObjectMap.get(spiName); + } +} diff --git a/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIObject.java b/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIObject.java new file mode 100644 index 00000000..f3daf53d --- /dev/null +++ b/wedpr-components/spi/src/main/java/com/webank/wedpr/components/spi/plugin/SPIObject.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.spi.plugin; + +public class SPIObject implements Comparable { + private SPIInfo spiInfo; + + public SPIObject(SPIInfo spiInfo) { + this.spiInfo = spiInfo; + } + + public SPIObject() {} + + public SPIInfo getSpiInfo() { + return spiInfo; + } + + public void setSpiInfo(SPIInfo spiInfo) { + this.spiInfo = spiInfo; + } + + @Override + public int compareTo(SPIObject o) { + return spiInfo.getPriority().compareTo(o.getSpiInfo().getPriority()); + } +} diff --git a/wedpr-components/task-plugin/api/build.gradle b/wedpr-components/task-plugin/api/build.gradle new file mode 100644 index 00000000..09e66a9e --- /dev/null +++ b/wedpr-components/task-plugin/api/build.gradle @@ -0,0 +1,16 @@ +// Apply the java-library plugin to add support for Java Library +plugins { + id 'java' + id 'com.github.sherter.google-java-format' +} +dependencies{ + compile project(":wedpr-core-utils") + compile project(":wedpr-components-spi") +} +googleJavaFormat { + //toolVersion = '1.7' + options style: 'AOSP' + source = sourceSets*.allJava + include '**/*.java' + //source = *.allJava +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/CommandExecutor.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/CommandExecutor.java new file mode 100644 index 00000000..a909a43b --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/CommandExecutor.java @@ -0,0 +1,108 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api; + +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskConfig; +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskExecutionContext; +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskResponse; +import com.webank.wedpr.components.task.plugin.api.shell.ShellBuilder; +import com.webank.wedpr.core.utils.Common; +import com.webank.wedpr.core.utils.Constant; +import com.webank.wedpr.core.utils.WeDPRException; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CommandExecutor implements WorkerExecutor { + private static final Logger logger = LoggerFactory.getLogger(CommandExecutor.class); + protected final CommandTaskExecutionContext context; + + public CommandExecutor(CommandTaskExecutionContext taskExecutionContext) { + this.context = taskExecutionContext; + } + + @Override + public CommandTaskResponse run(Object builder) throws Exception { + ShellBuilder shellBuilder = (ShellBuilder) builder; + // set the builder information + shellBuilder.context(context); + // set the system envs + if (!CommandTaskConfig.getSystemEnvFiles().isEmpty()) { + CommandTaskConfig.getSystemEnvFiles().forEach(shellBuilder::appendSystemEnv); + } + // set the environment + if (StringUtils.isNotBlank(context.getEnvironmentConfig())) { + shellBuilder.appendCustomEnv(context.getEnvironmentConfig()); + } + long remainTime = -1; + if (this.context.getTaskTimeoutMs() > 0) { + remainTime = (System.currentTimeMillis() - this.context.getStartTime()); + if (remainTime < 0) { + throw new WeDPRException("task execution timeout"); + } + } + // build the launcher + CommandTaskResponse taskResponse = new CommandTaskResponse(this.context.getTaskID()); + Process process = shellBuilder.build().execute(); + taskResponse.setProcess(process); + this.context.setProcess(process); + int processId = Common.getProcessId(process); + taskResponse.setProcessId(processId); + this.context.setProcessId(processId); + + logger.info("bootstrap process start, process id: {}", processId); + // wait for finish + boolean exitNormally = Boolean.FALSE; + if (remainTime > 0) { + exitNormally = process.waitFor(remainTime, TimeUnit.MILLISECONDS); + } else { + exitNormally = (process.waitFor() == 0 ? true : false); + } + if (exitNormally) { + taskResponse.setExitCode(process.exitValue()); + } else { + // kill the command + logger.error( + "process has failure, over the task timeout configuration {}, processId: {}, ready to kill", + context.getTaskTimeoutMs(), + processId); + kill(); + taskResponse.setExitCode(Constant.WEDPR_FAILED); + } + logger.info( + "execute process finished, executePath: {}, process: {}, exitCode: {}, exitNormally: {}", + this.context.getExecutePath(), + processId, + process.exitValue(), + exitNormally); + return taskResponse; + } + + @Override + public void kill() throws Exception { + if (this.context.getProcess() == null) { + return; + } + logger.info("Ready to kill process: {}", this.context.getProcessId()); + this.context.getProcess().destroy(); + if (!this.context + .getProcess() + .waitFor(CommandTaskConfig.getKillDefaultTimeoutSeconds(), TimeUnit.SECONDS)) { + this.context.getProcess().destroyForcibly(); + } + logger.info("Success kill process: {}", this.context.getProcessId()); + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilder.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilder.java new file mode 100644 index 00000000..132cc625 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilder.java @@ -0,0 +1,19 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api; + +public interface TaskBuilder { + public abstract TaskInterface createTask(TaskExecutionContext taskExecutionContext); +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilderFactory.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilderFactory.java new file mode 100644 index 00000000..538f7bb8 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskBuilderFactory.java @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api; + +import com.webank.wedpr.components.spi.plugin.SPIObject; + +public abstract class TaskBuilderFactory extends SPIObject { + + public abstract String getName(); + + public abstract TaskBuilder createTaskBuilder(); +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskExecutionContext.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskExecutionContext.java new file mode 100644 index 00000000..ddeaad6b --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskExecutionContext.java @@ -0,0 +1,46 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.io.Serializable; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class TaskExecutionContext implements Serializable { + private static final long serialVersionUID = -1L; + + // the taskID + protected String taskID; + // the workflowID of the task + protected String workflowID; + // the taskType + protected String taskType; + protected String taskParameters; + + // used to var-substitution + private Map parameterMap; + + // the startTime + protected Long startTime; + // -1 means never timeout + protected Long taskTimeoutMs = -1L; +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskInterface.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskInterface.java new file mode 100644 index 00000000..15057671 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskInterface.java @@ -0,0 +1,27 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api; + +public interface TaskInterface { + /** init the task */ + public abstract void init(); + + /** + * run the task + * + * @return the result + */ + public TaskResponse run(); +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskResponse.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskResponse.java new file mode 100644 index 00000000..14c49081 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/TaskResponse.java @@ -0,0 +1,32 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class TaskResponse { + protected String taskID; + protected Boolean success; + protected String resultDetail; + + public TaskResponse(String taskID) { + this.taskID = taskID; + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/WorkerExecutor.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/WorkerExecutor.java new file mode 100644 index 00000000..bf1417ac --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/WorkerExecutor.java @@ -0,0 +1,22 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.components.task.plugin.api; + +public interface WorkerExecutor { + public abstract TaskResponse run(Object param) throws Exception; + + public void kill() throws Exception; +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskConfig.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskConfig.java new file mode 100644 index 00000000..18518eff --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskConfig.java @@ -0,0 +1,79 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.model; + +import com.webank.wedpr.components.task.plugin.api.shell.ShellType; +import com.webank.wedpr.core.config.WeDPRConfig; +import com.webank.wedpr.core.utils.Common; +import com.webank.wedpr.core.utils.Constant; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.validation.constraints.NotNull; +import org.apache.commons.lang3.StringUtils; + +public class CommandTaskConfig { + private static final String DEFAULT_EXECUTE_DIR = + WeDPRConfig.apply("task.exec.dir", "/data/app"); + private static final String DEFAULT_LOG_BASE_PATH = + WeDPRConfig.apply("task.log.base.path", "."); + private static final List SYSTEM_ENV_FILES = + Arrays.stream( + Optional.ofNullable(WeDPRConfig.apply("task.system.env", "")) + .map(s -> s.split(",")) + .orElse(new String[0])) + .map(String::trim) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + private static final String SHELL_OUTPUT_FILE_PREFIX = + WeDPRConfig.apply("task.exec.output.prefix.file", "output"); + private static final Integer KILL_DEFAULT_TIMEOUT_SECONDS = + WeDPRConfig.apply("task.kill.timeout.seconds", 5); + + private static final Boolean ENABLE_SUDO = + WeDPRConfig.apply("task.exec.enable.sudo", Boolean.FALSE); + + private static final String SHELL_TYPE = + WeDPRConfig.apply("task.shell.type", ShellType.BASH.getType()); + + public static String getDefaultExecuteDir() { + return DEFAULT_EXECUTE_DIR; + } + + public static String getLogPath(@NotNull String appName) { + return Common.joinPath(DEFAULT_LOG_BASE_PATH, appName + Constant.LOG_POSTFIX); + } + // the system env files + public static List getSystemEnvFiles() { + return SYSTEM_ENV_FILES; + } + + public static Boolean getEnableSudo() { + return ENABLE_SUDO; + } + + public static String getOutputFile(String appName) { + return SHELL_OUTPUT_FILE_PREFIX + "_" + appName + Constant.LOG_POSTFIX; + } + + public static Integer getKillDefaultTimeoutSeconds() { + return KILL_DEFAULT_TIMEOUT_SECONDS; + } + + public static String getShellType() { + return SHELL_TYPE; + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskExecutionContext.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskExecutionContext.java new file mode 100644 index 00000000..409838bc --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskExecutionContext.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.webank.wedpr.components.task.plugin.api.TaskExecutionContext; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class CommandTaskExecutionContext extends TaskExecutionContext { + + // the appName + private String appName; + + // the task executePath + private String executePath = CommandTaskConfig.getDefaultExecuteDir(); + // the logPath + private String logPath = CommandTaskConfig.getLogPath(appName); + // the processInformation path + private String processInfoPath; + + // the environmentConfig + private String environmentConfig; + // the executeUser + private String executeUser; + + // the cpu-quota + private Integer cpuQuota = null; + // the max memory + private Integer maxMemory = null; + + // use sudo or not + private Boolean useSudo = Boolean.FALSE; + // running in background or not + private Boolean runningInBackground = Boolean.FALSE; + + // transient + @JsonIgnore private transient Process process; + @JsonIgnore private transient int processId; +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskResponse.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskResponse.java new file mode 100644 index 00000000..083da734 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/model/CommandTaskResponse.java @@ -0,0 +1,33 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.model; + +import com.webank.wedpr.components.task.plugin.api.TaskResponse; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CommandTaskResponse extends TaskResponse { + private int processId; + private Process process; + private Integer exitCode; + + public CommandTaskResponse(String taskID) { + super(taskID); + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/parameters/Parameters.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/parameters/Parameters.java new file mode 100644 index 00000000..34361499 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/parameters/Parameters.java @@ -0,0 +1,21 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.parameters; + +/** interface used to check and handle the parameter */ +public interface Parameters { + // check the parameter + public void checkParameters(); +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellBuilderImpl.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellBuilderImpl.java new file mode 100644 index 00000000..fc60f65b --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellBuilderImpl.java @@ -0,0 +1,220 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.components.task.plugin.api.shell; + +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskConfig; +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskExecutionContext; +import com.webank.wedpr.core.utils.Common; +import com.webank.wedpr.core.utils.FileUtils; +import com.webank.wedpr.core.utils.PropertiesHelper; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.*; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BaseShellBuilderImpl< + T extends BaseShellBuilderImpl, S extends ShellLauncher> + implements ShellBuilder { + private static final Logger logger = LoggerFactory.getLogger(BaseShellBuilderImpl.class); + + protected List systemEnvFiles = new ArrayList<>(); + protected List environmentConfigs = new ArrayList<>(); + protected List scripts = new ArrayList<>(); + + protected CommandTaskExecutionContext context; + + protected abstract String shellHeader(); + + protected abstract String shellPostfix(); + + protected abstract String shellType(); + + @Override + public T newBuilder(T builder) { + T populatedBuilder = newBuilder(); + populatedBuilder.systemEnvFiles = builder.systemEnvFiles; + populatedBuilder.context = builder.context; + return populatedBuilder; + } + + @Override + public T context(CommandTaskExecutionContext context) { + this.context = context; + return (T) this; + } + + @Override + public T appendSystemEnv(String envFile) { + if (StringUtils.isBlank(envFile)) { + return (T) this; + } + this.systemEnvFiles.add(envFile); + return (T) this; + } + + @Override + public T appendScript(String script) { + this.scripts.add(script); + return (T) this; + } + + public T appendCustomEnv(String envConfig) { + this.environmentConfigs.add(envConfig); + return (T) this; + } + + protected List generateLaunchCommand() { + List commands = new ArrayList<>(); + if (this.context.getRunningInBackground()) { + commands.add("nohup"); + } + if (!this.context.getUseSudo() || !CommandTaskConfig.getEnableSudo()) { + commands.addAll(generateLaunchCommandInNormalMode()); + } else { + commands.addAll(generateLaunchCommandInSudoMode()); + } + // TODO: define the output using context parameter + if (this.context.getRunningInBackground()) { + commands.add( + String.format( + ">%s 2>&1 &", + CommandTaskConfig.getOutputFile(this.context.getAppName()))); + } + return commands; + } + + protected Path getShellScriptPath() { + return Paths.get(this.context.getExecutePath(), this.context.getAppName() + shellPostfix()); + } + + protected void generateShellScript() throws Exception { + List scriptContents = new ArrayList<>(); + // add shellHeader information + scriptContents.add(shellHeader()); + // add baseDir information + scriptContents.add(ShellConstant.BASE_DIR_CMD); + // add system env commands + scriptContents.addAll(generateSystemEnvScript()); + // add environment commands + scriptContents.addAll(environmentConfigs); + // add the shell + scriptContents.add(generateShellContent()); + String scriptData = + scriptContents.stream().collect(Collectors.joining(System.lineSeparator())); + // generate and write the shell-script content + FileUtils.createExecutableFile(getShellScriptPath()); + Files.write(getShellScriptPath(), scriptData.getBytes(), StandardOpenOption.APPEND); + logger.info( + "generateShellScript success, file: {}, content: {}", + getShellScriptPath().toString(), + scriptData); + } + + private List generateLaunchCommandInNormalMode() { + List commands = new ArrayList<>(); + commands.add(shellType()); + commands.add(getShellScriptPath().toString()); + logger.info("generateLaunchCommandInNormalMode, command: {}", String.join(" ", commands)); + return commands; + } + + private List generateLaunchCommandInSudoMode() { + // execute command with resource limit + List commands; + if (!PropertiesHelper.getValue( + this.context.getParameterMap(), + ShellConstant.RESOURCE_LIMIT_PROPERTY, + false, + Boolean.FALSE)) { + commands = generateSudoModeCommandSettingWithoutResourceLimit(); + } else { + commands = generateSudoModeCommandSettingWithResourceLimit(); + } + commands.addAll(generateLaunchCommandInNormalMode()); + logger.info("generateLaunchCommandInSudoMode, command: {}", String.join(" ", commands)); + return commands; + } + + private List generateSudoModeCommandSettingWithResourceLimit() { + List commands = new ArrayList<>(); + commands.add(ShellConstant.RESOURCE_LIMIT_CMD); + if (this.context.getCpuQuota() != null) { + commands.add("-p"); + // without limit + if (this.context.getCpuQuota() <= 0) { + commands.add(String.format("%s=", ShellConstant.CPU_QUOTA)); + } else { + commands.add( + String.format( + "%s=%s%%", ShellConstant.CPU_QUOTA, this.context.getCpuQuota())); + } + } + + if (this.context.getMaxMemory() != null) { + commands.add("-p"); + // without limit + if (this.context.getMaxMemory() <= 0) { + commands.add( + String.format("%s=%s", ShellConstant.MEMORY_LIMIT, ShellConstant.INFINITY)); + } else { + commands.add( + String.format( + "%s=%s", ShellConstant.MEMORY_LIMIT, this.context.getMaxMemory())); + } + } + if (StringUtils.isNotBlank(this.context.getExecuteUser())) { + commands.add(String.format("--uid=%s", this.context.getExecuteUser())); + } + return commands; + } + + private List generateSudoModeCommandSettingWithoutResourceLimit() { + List commands = new ArrayList<>(); + commands.add(ShellConstant.SUDO_COMMAND); + if (StringUtils.isNotBlank(this.context.getExecuteUser())) { + commands.add("-u"); + commands.add(this.context.getExecuteUser()); + } + return commands; + } + + protected List generateSystemEnvScript() { + if (systemEnvFiles.isEmpty()) { + return Collections.emptyList(); + } + return systemEnvFiles + .stream() + .map(systemEnvFile -> "source " + systemEnvFile) + .collect(Collectors.toList()); + } + + private String generateShellContent() { + if (scripts.isEmpty()) { + return StringUtils.EMPTY; + } + String scriptsContent = + scripts.stream() + .collect(Collectors.joining(System.lineSeparator())) + .replaceAll(ShellConstant.WINDOWS_LINE_SPLITTER, System.lineSeparator()); + // substitutor with parameterMaps + return Common.substitutorVarsWithParameters(scriptsContent, this.context.getParameterMap()); + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellLauncher.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellLauncher.java new file mode 100644 index 00000000..20ff1360 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/BaseShellLauncher.java @@ -0,0 +1,40 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.shell; + +import java.io.File; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BaseShellLauncher implements ShellLauncher { + private static final Logger logger = LoggerFactory.getLogger(BaseShellLauncher.class); + protected final List commands; + protected final String workingDir; + + public BaseShellLauncher(List commands, String workingDir) { + this.commands = commands; + this.workingDir = workingDir; + } + + @Override + public Process execute() throws Exception { + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.directory(new File(workingDir)); + processBuilder.command(commands); + logger.info("Begin to execute shell command: [{}]", String.join(" ", commands)); + return processBuilder.start(); + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilder.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilder.java new file mode 100644 index 00000000..8c76bef6 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilder.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.components.task.plugin.api.shell; + +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskExecutionContext; + +public interface ShellBuilder, S extends ShellLauncher> { + public T newBuilder(); + + public T newBuilder(T builder); + + public T context(CommandTaskExecutionContext context); + + public T appendSystemEnv(String envFile); + + public T appendCustomEnv(String envFile); + + public T appendScript(String script); + + public S build(); +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilderFactory.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilderFactory.java new file mode 100644 index 00000000..ea3fa929 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellBuilderFactory.java @@ -0,0 +1,46 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.shell; + +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskConfig; +import com.webank.wedpr.components.task.plugin.api.shell.bash.BashBuilder; +import com.webank.wedpr.core.utils.WeDPRException; +import lombok.SneakyThrows; + +public class ShellBuilderFactory { + private static ShellType shellType; + + static { + loadShellType(); + } + + @SneakyThrows(Exception.class) + private static void loadShellType() { + shellType = ShellType.deserialize(CommandTaskConfig.getShellType()); + if (shellType == null) { + throw new WeDPRException( + "Not supported shell type " + CommandTaskConfig.getShellType()); + } + } + + @SneakyThrows(Exception.class) + public static ShellBuilder create() { + if (shellType.equals(ShellType.BASH)) { + return new BashBuilder(); + } + throw new WeDPRException( + "Create ShellBuilder failed for not support shell type: " + shellType.getType()); + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellCommandExecutor.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellCommandExecutor.java new file mode 100644 index 00000000..6945ff4f --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellCommandExecutor.java @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.shell; + +import com.webank.wedpr.components.task.plugin.api.CommandExecutor; +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskExecutionContext; + +public class ShellCommandExecutor extends CommandExecutor { + public ShellCommandExecutor(CommandTaskExecutionContext taskExecutionContext) { + super(taskExecutionContext); + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellConstant.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellConstant.java new file mode 100644 index 00000000..a04b66cb --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellConstant.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.components.task.plugin.api.shell; + +public class ShellConstant { + public static String BASE_DIR_CMD = "BASEDIR=$(cd `dirname $0`; pwd) && cd ${BASEDIR}"; + public static String WINDOWS_LINE_SPLITTER = "\\r\\n"; + public static String RESOURCE_LIMIT_PROPERTY = "resource.limit.enable"; + public static String SUDO_COMMAND = "sudo"; + public static String RESOURCE_LIMIT_CMD = "sudo systemd-run -q --scope"; + public static String CPU_QUOTA = "CPUQuota"; + public static String MEMORY_LIMIT = "MemoryLimit"; + public static String INFINITY = "infinity"; + + public static String BASH_HEADER = "#!/bin/bash"; + public static String SHELL_POSTFIX = ".sh"; +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellLauncher.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellLauncher.java new file mode 100644 index 00000000..ad561519 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellLauncher.java @@ -0,0 +1,21 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.shell; + +/** interface used to execute the shell commands */ +public interface ShellLauncher { + + public Process execute() throws Exception; +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellType.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellType.java new file mode 100644 index 00000000..63539652 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/ShellType.java @@ -0,0 +1,44 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ + +package com.webank.wedpr.components.task.plugin.api.shell; + +import org.apache.commons.lang3.StringUtils; + +public enum ShellType { + BASH("bash"); + + private final String type; + + ShellType(String type) { + this.type = type; + } + + public String getType() { + return this.type; + } + + public static ShellType deserialize(String type) { + if (StringUtils.isBlank(type)) { + return null; + } + for (ShellType shellType : ShellType.values()) { + if (shellType.type.compareToIgnoreCase(type) == 0) { + return shellType; + } + } + return null; + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashBuilder.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashBuilder.java new file mode 100644 index 00000000..6b43a08b --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashBuilder.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.shell.bash; + +import com.webank.wedpr.components.task.plugin.api.shell.BaseShellBuilderImpl; +import com.webank.wedpr.components.task.plugin.api.shell.ShellConstant; +import java.util.List; +import lombok.SneakyThrows; + +public class BashBuilder extends BaseShellBuilderImpl { + @Override + public BashBuilder newBuilder() { + return new BashBuilder(); + } + + @SneakyThrows(Exception.class) + @Override + public BashLauncher build() { + generateShellScript(); + List commands = generateLaunchCommand(); + return new BashLauncher(commands, this.context.getExecutePath()); + } + + @Override + protected String shellHeader() { + return ShellConstant.BASH_HEADER; + } + + @Override + protected String shellPostfix() { + return ShellConstant.SHELL_POSTFIX; + } + + @Override + protected String shellType() { + return "bash"; + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashLauncher.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashLauncher.java new file mode 100644 index 00000000..f11fafb4 --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/api/shell/bash/BashLauncher.java @@ -0,0 +1,24 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.api.shell.bash; + +import com.webank.wedpr.components.task.plugin.api.shell.BaseShellLauncher; +import java.util.List; + +public class BashLauncher extends BaseShellLauncher { + public BashLauncher(List commands, String workingDir) { + super(commands, workingDir); + } +} diff --git a/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/loader/TaskPluginLoader.java b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/loader/TaskPluginLoader.java new file mode 100644 index 00000000..2cd5a4ac --- /dev/null +++ b/wedpr-components/task-plugin/api/src/main/java/com/webank/wedpr/components/task/plugin/loader/TaskPluginLoader.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.loader; + +import com.webank.wedpr.components.spi.plugin.SPILoader; +import com.webank.wedpr.components.task.plugin.api.TaskBuilder; +import com.webank.wedpr.components.task.plugin.api.TaskBuilderFactory; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TaskPluginLoader { + private static Logger logger = LoggerFactory.getLogger(TaskPluginLoader.class); + private static final Map taskBuilderMap = new HashMap<>(); + private static AtomicBoolean loaded = new AtomicBoolean(false); + + static { + load(); + } + + public static void load() { + if (!loaded.compareAndSet(false, true)) { + logger.warn("The task plugins have already been loaded!"); + return; + } + SPILoader taskBuilderFactorySPILoader = + new SPILoader<>(TaskBuilderFactory.class); + // create the taskBuilder + for (String name : taskBuilderFactorySPILoader.getSpiObjectMap().keySet()) { + TaskBuilderFactory factory = taskBuilderFactorySPILoader.getSPIObjectByName(name); + if (factory == null) { + continue; + } + taskBuilderMap.put(factory.getName(), factory.createTaskBuilder()); + logger.info( + "register task plugin success, name: {}, classInfo: {}", + name, + factory.getClass().getSimpleName()); + } + } + + public static Map getTaskBuilderMap() { + return taskBuilderMap; + } + + public static TaskBuilder getTaskBuilderByName(String name) { + if (!taskBuilderMap.containsKey(name)) { + return null; + } + return taskBuilderMap.get(name); + } +} diff --git a/wedpr-components/task-plugin/shell/build.gradle b/wedpr-components/task-plugin/shell/build.gradle new file mode 100644 index 00000000..d30f09e0 --- /dev/null +++ b/wedpr-components/task-plugin/shell/build.gradle @@ -0,0 +1,15 @@ +// Apply the java-library plugin to add support for Java Library +plugins { + id 'java' + id 'com.github.sherter.google-java-format' +} +dependencies{ + compile project(":wedpr-components-task-plugin-api") +} +googleJavaFormat { + //toolVersion = '1.7' + options style: 'AOSP' + source = sourceSets*.allJava + include '**/*.java' + //source = *.allJava +} diff --git a/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellParameters.java b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellParameters.java new file mode 100644 index 00000000..de360d01 --- /dev/null +++ b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellParameters.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.shell; + +import com.webank.wedpr.components.task.plugin.api.parameters.Parameters; +import com.webank.wedpr.core.utils.Common; +import com.webank.wedpr.core.utils.ObjectMapperFactory; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; + +@Getter +@Setter +public class ShellParameters implements Parameters { + // the script code + private String code; + + @Override + // check the parameter + public void checkParameters() { + Common.requireNonEmpty("code", this.code); + } + + public static ShellParameters deserialize(String parameter) throws Exception { + if (StringUtils.isBlank(parameter)) { + return null; + } + return ObjectMapperFactory.getObjectMapper().readValue(parameter, ShellParameters.class); + } +} diff --git a/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTask.java b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTask.java new file mode 100644 index 00000000..90ec9417 --- /dev/null +++ b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTask.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.shell; + +import com.webank.wedpr.components.task.plugin.api.TaskExecutionContext; +import com.webank.wedpr.components.task.plugin.api.TaskInterface; +import com.webank.wedpr.components.task.plugin.api.TaskResponse; +import com.webank.wedpr.components.task.plugin.api.model.CommandTaskExecutionContext; +import com.webank.wedpr.components.task.plugin.api.shell.ShellBuilder; +import com.webank.wedpr.components.task.plugin.api.shell.ShellBuilderFactory; +import com.webank.wedpr.components.task.plugin.api.shell.ShellCommandExecutor; +import com.webank.wedpr.core.utils.WeDPRException; +import lombok.SneakyThrows; + +public class ShellTask implements TaskInterface { + protected final CommandTaskExecutionContext context; + protected final ShellParameters parameters; + protected final ShellCommandExecutor executor; + + @SneakyThrows(Exception.class) + public ShellTask(TaskExecutionContext taskExecutionContext) { + this.context = (CommandTaskExecutionContext) taskExecutionContext; + // parse and check the task parameter + this.parameters = ShellParameters.deserialize(this.context.getTaskParameters()); + if (this.parameters == null) { + throw new WeDPRException( + "Create ShellTask failed for invalid taskParameter: " + + this.context.getTaskParameters()); + } + this.parameters.checkParameters(); + // create the executor + this.executor = new ShellCommandExecutor(this.context); + } + + // do nothing here + @Override + public void init() {} + + @SneakyThrows(Exception.class) + @Override + public TaskResponse run() { + ShellBuilder shellBuilder = ShellBuilderFactory.create(); + shellBuilder.appendScript(parameters.getCode()); + return this.executor.run(shellBuilder); + } +} diff --git a/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilder.java b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilder.java new file mode 100644 index 00000000..16c1e478 --- /dev/null +++ b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilder.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.shell; + +import com.webank.wedpr.components.task.plugin.api.TaskBuilder; +import com.webank.wedpr.components.task.plugin.api.TaskExecutionContext; +import com.webank.wedpr.components.task.plugin.api.TaskInterface; + +public class ShellTaskBuilder implements TaskBuilder { + @Override + public TaskInterface createTask(TaskExecutionContext context) { + return new ShellTask(context); + } +} diff --git a/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilderFactory.java b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilderFactory.java new file mode 100644 index 00000000..b6df60d1 --- /dev/null +++ b/wedpr-components/task-plugin/shell/src/main/java/com/webank/wedpr/components/task/plugin/shell/ShellTaskBuilderFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.webank.wedpr.components.task.plugin.shell; + +import com.google.auto.service.AutoService; +import com.webank.wedpr.components.task.plugin.api.TaskBuilder; +import com.webank.wedpr.components.task.plugin.api.TaskBuilderFactory; + +@AutoService(TaskBuilderFactory.class) +public class ShellTaskBuilderFactory extends TaskBuilderFactory { + public static final String NAME = "SHELL"; + + @Override + public String getName() { + return NAME; + } + + @Override + public TaskBuilder createTaskBuilder() { + return new ShellTaskBuilder(); + } +} diff --git a/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Common.java b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Common.java index d9f52332..cb90196f 100644 --- a/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Common.java +++ b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Common.java @@ -28,8 +28,13 @@ import java.util.Objects; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.StringSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Common { + private static final Logger logger = LoggerFactory.getLogger(Common.class); + @SneakyThrows(WeDPRException.class) public static void requireNonEmpty(String fieldName, String fieldValue) { if (StringUtils.isBlank(fieldValue) @@ -122,4 +127,26 @@ public static String timeToString(LocalDateTime time) { public static String getCurrentTime() { return timeToString(LocalDateTime.now()); } + + public static int getProcessId(Process process) throws WeDPRException { + try { + Field field = process.getClass().getField(Constant.PID_FIELD); + field.setAccessible(true); + return field.getInt(process); + } catch (Exception e) { + logger.warn("getProcessId failed, error: ", e); + throw new WeDPRException("getProcessId failed for " + e.getMessage(), e); + } + } + + // replace the templateContent with vars + public static String substitutorVarsWithParameters( + String templateContent, Map parameterMap) { + // support ${datetime} + if (!parameterMap.containsKey(Constant.DATE_VAR)) { + parameterMap.put(Constant.DATE_VAR, getCurrentTime()); + } + StringSubstitutor stringSubstitutor = new StringSubstitutor(parameterMap); + return stringSubstitutor.replace(templateContent); + } } diff --git a/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Constant.java b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Constant.java index e13af54b..c133abe3 100644 --- a/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Constant.java +++ b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/Constant.java @@ -45,6 +45,7 @@ public class Constant { public static final String SITE_END_LOGIN_URL = WEDPR_API_PREFIX + "/login"; public static final String ADMIN_END_LOGIN_URL = WEDPR_API_PREFIX + "/admin/login"; + public static final String REGISTER_URL = WEDPR_API_PREFIX + "/register"; public static final String USER_PUBLICKEY_URL = WEDPR_API_PREFIX + "/pub"; public static final String IMAGE_CODE_URL = WEDPR_API_PREFIX + "/image-code"; @@ -56,4 +57,8 @@ public class Constant { public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd"; public static final String DEFAULT_TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss"; public static final String DEFAULT_INIT_GROUP_ID = "1000000000000000"; + + public static final String PID_FIELD = "pid"; + public static final String DATE_VAR = "datetime"; + public static final String LOG_POSTFIX = ".log"; } diff --git a/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/FileUtils.java b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/FileUtils.java new file mode 100644 index 00000000..0d6e3c26 --- /dev/null +++ b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/FileUtils.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017-2025 [webank-wedpr] + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + */ +package com.webank.wedpr.core.utils; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; +import javax.validation.constraints.NotNull; + +public class FileUtils { + public static final Set EXECUTABLE_PERMISSION = + PosixFilePermissions.fromString("rwxr-xr-x"); + + public static void createExecutableFile(@NotNull Path path) throws Exception { + Files.createFile(path); + Files.setPosixFilePermissions(path, EXECUTABLE_PERMISSION); + } +} diff --git a/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/PropertiesHelper.java b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/PropertiesHelper.java index 76c54551..aacedba6 100644 --- a/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/PropertiesHelper.java +++ b/wedpr-core/utils/src/main/java/com/webank/wedpr/core/utils/PropertiesHelper.java @@ -16,11 +16,19 @@ package com.webank.wedpr.core.utils; import java.nio.charset.StandardCharsets; +import java.util.Map; import java.util.Properties; import lombok.SneakyThrows; +import org.apache.commons.collections.MapUtils; public class PropertiesHelper { + @SneakyThrows + public static T getValue( + Map config, String key, boolean required, T defaultValue) { + return getValue(MapUtils.toProperties(config), key, required, defaultValue); + } + @SneakyThrows public static T getValue(Properties config, String key, boolean required, T defaultValue) { if (config == null) {