fix bugs

kevinw66 · Aug 8, 2024 · b7307e8 · b7307e8
1 parent d65a929
commit b7307e8
Show file tree

Hide file tree

Showing 8 changed files with 81 additions and 31 deletions.
diff --git a/...op-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java b/...op-manager-common/src/main/java/org/apache/bigtop/manager/common/constants/Constants.java
@@ -42,14 +42,21 @@ private Constants() {
      */
     public static final String ALL_HOST_KEY = "all";
 
+    /**
+     * permission 644
+     */
+    public static final String PERMISSION_644 = "rw-r--r--";
+
     /**
      * permission 755
      */
     public static final String PERMISSION_755 = "rwxr-xr-x";
+
     /**
-     * permission 644
+     * permission 775
      */
-    public static final String PERMISSION_644 = "rw-r--r--";
+    public static final String PERMISSION_775 = "rwxrwxr-x";
+
     /**
      * permission 777
      */

diff --git a/...-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml b/...-server/src/main/resources/stacks/bigtop/3.3.0/services/hdfs/configuration/hadoop-env.xml
@@ -86,33 +86,22 @@
 
 USER="$(whoami)"
 export JAVA_HOME=${java_home!}
-
 export HADOOP_HOME=${hadoop_home!}
-
 export HADOOP_COMMON_HOME=${hadoop_home!}
-
 export HADOOP_HDFS_HOME=${hadoop_hdfs_home!}
-
 export HADOOP_CONF_DIR=${hadoop_conf_dir!}
-
 export HADOOP_HEAPSIZE_MAX=${hadoop_heapsize_max}
-
 export HADOOP_HEAPSIZE_MIN=${hadoop_heapsize_min}
 
-export HADOOP_JAAS_DEBUG=true
-
 # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"
 
 <#noparse>
 export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
 </#noparse>
 
 export HADOOP_LOG_DIR=${hadoop_log_dir_prefix}/$USER
-
 export HADOOP_PID_DIR=${hadoop_pid_dir_prefix}/$USER
-
 export HADOOP_ROOT_LOGGER=${hadoop_root_logger}
-
 export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir}
 
 ]]>

diff --git a/...ger-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml b/...ger-server/src/main/resources/stacks/bigtop/3.3.0/services/tez/configuration/tez-site.xml
@@ -57,7 +57,7 @@
     <property>
         <name>tez.staging-dir</name>
         <display-name>TEZ Staging directory</display-name>
-        <value>/tmp/${tez_user!}/staging</value>
+        <value>/tmp/${user.name}/staging</value>
         <description>The staging dir used while submitting DAGs</description>
     </property>
     <property>

diff --git a/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml b/bigtop-manager-stack/bigtop-manager-stack-bigtop/pom.xml
@@ -58,6 +58,14 @@
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-log4j12</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.reload4j</groupId>
+                    <artifactId>reload4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -74,6 +82,14 @@
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-log4j12</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.reload4j</groupId>
+                    <artifactId>reload4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -90,6 +106,14 @@
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-log4j12</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.reload4j</groupId>
+                    <artifactId>reload4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
     </dependencies>

diff --git a/...ger-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java b/...ger-stack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/utils/HdfsUtil.java
@@ -38,28 +38,38 @@
 @Data
 @Slf4j
 public class HdfsUtil {
+    public static void createDirectory(String user, String directory) {
+        UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+        try {
+            ugi.doAs((PrivilegedAction<Void>) () -> {
+                try (FileSystem fs = getFileSystem()) {
+                    // Create dest dir if not exist
+                    Path destDirPath = new Path(directory);
+                    if (!fs.exists(destDirPath)) {
+                        fs.mkdirs(destDirPath);
+                    }
+                } catch (Exception e) {
+                    log.error("Error while creating directory on hdfs", e);
+                    throw new StackException(e);
+                }
+
+                return null;
+            });
+        } catch (Exception e) {
+            log.error("Error while creating directory on hdfs", e);
+            throw new StackException(e);
+        }
+    }
 
     public static void uploadFile(String user, String localFilePath, String destDir) {
         uploadFile(user, localFilePath, destDir, null);
     }
 
     public static void uploadFile(String user, String localFilePath, String destDir, String destFilename) {
-        Configuration conf = new Configuration();
-        conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
-        conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));
-
-        List<String> namenodeList = LocalSettings.hosts("namenode");
-        if (CollectionUtils.isEmpty(namenodeList)) {
-            String msg = "No namenode found in the cluster";
-            log.error(msg);
-            throw new StackException(msg);
-        }
-
-        String hdfsUri = MessageFormat.format("hdfs://{0}:8020", namenodeList.get(0));
         UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
         try {
             ugi.doAs((PrivilegedAction<Void>) () -> {
-                try (FileSystem fs = FileSystem.get(new URI(hdfsUri), conf)) {
+                try (FileSystem fs = getFileSystem()) {
                     // Create dest dir if not exist
                     Path destDirPath = new Path(destDir);
                     if (!fs.exists(destDirPath)) {
@@ -81,4 +91,20 @@ public static void uploadFile(String user, String localFilePath, String destDir,
             throw new StackException(e);
         }
     }
+
+    private static FileSystem getFileSystem() throws Exception {
+        Configuration conf = new Configuration();
+        conf.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
+        conf.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));
+
+        List<String> namenodeList = LocalSettings.hosts("namenode");
+        if (CollectionUtils.isEmpty(namenodeList)) {
+            String msg = "No namenode found in the cluster";
+            log.error(msg);
+            throw new StackException(msg);
+        }
+
+        String hdfsUri = MessageFormat.format("hdfs://{0}:8020", namenodeList.get(0));
+        return FileSystem.get(new URI(hdfsUri), conf);
+    }
 }
diff --git a/...ck-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java b/...ck-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/hdfs/HdfsSetup.java
@@ -83,7 +83,7 @@ public static ShellResult config(Params params, String componentName) {
         LinuxFileUtils.createDirectories(
                 hdfsParams.getDfsDataDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true);
         LinuxFileUtils.createDirectories(
-                hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true);
+                hdfsParams.getHadoopLogDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_775, true);
         LinuxFileUtils.createDirectories(
                 hdfsParams.getHadoopPidDir(), hdfsUser, hdfsGroup, Constants.PERMISSION_755, true);
 
@@ -150,6 +150,10 @@ public static ShellResult config(Params params, String componentName) {
                 Constants.PERMISSION_644,
                 hdfsParams.getGlobalParamsMap());
 
+        // log.info("Creating /apps on hdfs");
+        // HdfsUtil.createDirectory(hdfsUser, "/apps");
+
+        log.info("Successfully configured HDFS");
         return ShellResult.success("HDFS Configure success!");
     }
 

diff --git a/...tack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java b/...tack-bigtop/src/main/java/org/apache/bigtop/manager/stack/bigtop/v3_3_0/tez/TezSetup.java
@@ -30,7 +30,7 @@
 
 import java.text.MessageFormat;
 
-import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_644;
+import static org.apache.bigtop.manager.common.constants.Constants.PERMISSION_755;
 
 @Slf4j
 @NoArgsConstructor(access = AccessLevel.PRIVATE)
@@ -61,7 +61,7 @@ public static ShellResult config(Params params) {
                 MessageFormat.format("{0}/tez-env.sh", confDir),
                 tezUser,
                 tezGroup,
-                PERMISSION_644,
+                PERMISSION_755,
                 tezParams.getGlobalParamsMap());
 
         // maybe we should upload tez.tar.gz to HDFS here?

diff --git a/...ager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java b/...ager-stack-core/src/main/java/org/apache/bigtop/manager/stack/core/hook/AbstractHook.java
@@ -58,7 +58,7 @@ private void addUserAndGroup() {
             }
 
             log.info(
-                    "Adding user: {} to group: {} and groups: [{}]",
+                    "Adding user: {} to primary group: {} and supplementary groups: [{}]",
                     user.getKey(),
                     userGroup,
                     String.join(",", groups));