[PLAT-15074] Install node agents on nodes for the marked universes fo…

…r on-prem providers

Summary:
Install node agent in YB_HOME path as user-level systemd.

For CSPs, we still want to use root node-agent until the YNP migration is done.

Test Plan:
1. Created universes with aws and on-prem fully manual providers.
2. Verified that background node agent installed node agent on on-prem manual universe.
3. Verified that create universe still works on aws with node agent.

Itests will also verify.

Reviewers: svarshney, amalyshev, sanketh

Reviewed By: amalyshev

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D38574

managed/node-agent/resources/node-agent-installer.sh

@@ -20,7 +20,6 @@ SKIP_VERIFY_CERT=""
 DISABLE_EGRESS="false"
 SILENT_INSTALL="false"
 AIRGAP_INSTALL="false"
-USER_SCOPED_UNIT="false"
 SKIP_PACKAGE_DOWNLOAD="false"
 CERT_DIR=""
 CUSTOMER_ID=""
@@ -155,22 +154,7 @@ uninstall_node_agent() {
   local NODE_AGENT_UUID=""
   NODE_AGENT_UUID="$(grep -o '"uuid":"[^"]*"' "$RESPONSE_FILE" | cut -d: -f2 | tr -d '"')"
   rm -rf "$RESPONSE_FILE"
-  local RUNNING=""
-  RUNNING=$(systemctl list-units | grep -F yb-node-agent.service)
-  # If not found, check in user-level units
-  if [ -z "$RUNNING" ]; then
-    RUNNING=$(systemctl --user list-units | grep -F yb-node-agent.service)
-  fi
-
-  if [ -n "$RUNNING" ]; then
-     if [ "$USER_SCOPED_UNIT" = "false" ]; then
-      run_as_super_user systemctl stop yb-node-agent
-      run_as_super_user systemctl disable yb-node-agent
-    else
-      systemctl --user stop yb-node-agent
-      systemctl --user disable yb-node-agent
-    fi
-  fi
+  stop_systemd_service
   if [ -n "$NODE_AGENT_UUID" ]; then
     local STATUS_CODE=""
     STATUS_CODE=$(curl -s ${SKIP_VERIFY_CERT:+ "-k"} -w "%{http_code}" -L --request DELETE \
@@ -308,7 +292,38 @@ modify_selinux() {
   set -e
 }
 
+stop_systemd_service() {
+  local UNIT_FILE_PRESENT=""
+  local USER_SCOPED_UNIT=""
+  set +e
+  UNIT_FILE_PRESENT=$(systemctl list-units | grep -F yb-node-agent.service)
+  # If not found, check in user-level units
+  if [ -z "$UNIT_FILE_PRESENT" ]; then
+    UNIT_FILE_PRESENT=$(systemctl --user list-units | grep -F yb-node-agent.service)
+    if [ -n "$UNIT_FILE_PRESENT" ]; then
+      USER_SCOPED_UNIT="true"
+    fi
+  else
+    USER_SCOPED_UNIT="false"
+  fi
+  if [ -n "$UNIT_FILE_PRESENT" ]; then
+    if [ "$USER_SCOPED_UNIT" = "false" ] && [ "$SUDO_ACCESS" = "true" ]; then
+      run_as_super_user systemctl stop yb-node-agent
+      run_as_super_user systemctl disable yb-node-agent
+    elif [ "$USER_SCOPED_UNIT" = "true" ]; then
+      systemctl --user stop yb-node-agent
+      systemctl --user disable yb-node-agent
+    fi
+  fi
+  set -e
+}
+
 install_systemd_service() {
+  local USER_SCOPED_UNIT="false"
+  if [ "$SUDO_ACCESS" = "false" ]; then
+    USER_SCOPED_UNIT="true"
+    SYSTEMD_PATH="$INSTALL_USER_HOME/.config/systemd/user"
+  fi
   if [ "$SE_LINUX_STATUS" = "Enforcing" ]; then
     modify_selinux
   fi
@@ -434,10 +449,6 @@ err_msg() {
 main() {
   echo "* Starting YB Node Agent $COMMAND."
   if [ "$COMMAND" = "install_service" ]; then
-    if [ "$SUDO_ACCESS" = "false" ]; then
-      USER_SCOPED_UNIT="true"
-      SYSTEMD_PATH="$INSTALL_USER_HOME/.config/systemd/user"
-    fi
     install_systemd_service
   elif [ "$COMMAND" = "upgrade" ]; then
     extract_package > /dev/null
@@ -529,23 +540,7 @@ main() {
         exit 1
       fi
       # Disable existing node-agent if sudo access is available.
-      local RUNNING=""
-      set +e
-      RUNNING=$(systemctl list-units | grep -F yb-node-agent.service)
-      # If not found, check in user-level units
-      if [ -z "$RUNNING" ]; then
-        RUNNING=$(systemctl --user list-units | grep -F yb-node-agent.service)
-      fi
-      if [ -n "$RUNNING" ]; then
-        if [ "$USER_SCOPED_UNIT" = "false" ] && [ "$SUDO_ACCESS" = "true" ]; then
-          run_as_super_user systemctl stop yb-node-agent
-          run_as_super_user systemctl disable yb-node-agent
-        else
-          systemctl --user stop yb-node-agent
-          systemctl --user disable yb-node-agent
-        fi
-      fi
-      set -e
+      stop_systemd_service
       NODE_AGENT_CONFIG_ARGS+=(--disable_egress --id "$NODE_AGENT_ID" --customer_id "$CUSTOMER_ID" \
       --cert_dir "$CERT_DIR" --node_name "$NODE_NAME" --node_ip "$NODE_IP" \
       --node_port "$NODE_PORT" "${SKIP_VERIFY_CERT:+ "--skip_verify_cert"}")
@@ -755,10 +750,11 @@ check_sudo_access
 main
 
 if [ "$?" -eq 0 ] && [ "$COMMAND" = "install" ]; then
-  if [ "$SUDO_ACCESS" = "false" ]; then
+  if [ "$DISABLE_EGRESS" == "false" ]; then
     echo "You can install a systemd service on linux machines\
  by running $INSTALLER_NAME -c install_service --user yugabyte (Requires sudo access)."
   else
-     install_systemd_service
+    echo "Automatically installing systemd service for node agent installed by YBA."
+    install_systemd_service
   fi
 fi

managed/src/main/java/com/yugabyte/yw/commissioner/NodeAgentEnabler.java

@@ -234,6 +234,10 @@ private boolean isBackgroundInstallNodeAgentEnabled(Universe universe) {
     return isNodeAgentEnabled(
             universe,
             p -> {
+              Cluster primaryCluster = universe.getUniverseDetails().getPrimaryCluster();
+              if (primaryCluster.userIntent.useSystemd == false) {
+                return false;
+              }
               if (p.getCloudCode() != CloudType.onprem || p.getDetails().isSkipProvisioning()) {
                 // Do not include provider flag for cloud and fully manual onprem providers when the
                 // enabler is on.

managed/src/main/java/com/yugabyte/yw/commissioner/NodeAgentInstallerImpl.java

@@ -52,11 +52,6 @@ protected NodeAgentInstallerImpl(
 
   @Override
   public boolean install(UUID customerUuid, UUID universeUuid, NodeDetails nodeDetails) {
-    if (isOnPremFullyManual(universeUuid, nodeDetails)) {
-      // TODO Need to revisit this for on-prem installation once the user-level systemd changes are
-      // done.
-      return false;
-    }
     InstallNodeAgent task = AbstractTaskBase.createTask(InstallNodeAgent.class);
     task.initialize(createInstallParams(customerUuid, universeUuid, nodeDetails, false));
     waitForNodeAgent(task.install());
@@ -66,11 +61,6 @@ public boolean install(UUID customerUuid, UUID universeUuid, NodeDetails nodeDet
   @Override
   public boolean reinstall(
       UUID customerUuid, UUID universeUuid, NodeDetails nodeDetails, NodeAgent nodeAgent) {
-    if (isOnPremFullyManual(universeUuid, nodeDetails)) {
-      // TODO Need to revisit this for on-prem installation once the user-level systemd changes are
-      // done.
-      return false;
-    }
     State state = nodeAgent.getState();
     if (state == State.REGISTERING) {
       InstallNodeAgent task = AbstractTaskBase.createTask(InstallNodeAgent.class);
@@ -141,25 +131,20 @@ private InstallNodeAgent.Params createInstallParams(
     params.setUniverseUUID(universe.getUniverseUUID());
     params.nodeAgentInstallDir = installPath;
     params.nodeAgentPort = serverPort;
-    params.reinstall = reinstall;
     if (StringUtils.isNotEmpty(nodeDetails.sshUserOverride)) {
       params.sshUser = nodeDetails.sshUserOverride;
     }
+    if (provider.getCloudCode() == CloudType.onprem && provider.getDetails().skipProvisioning) {
+      params.sudoAccess = false;
+      params.nodeAgentInstallDir = provider.getYbHome();
+    } else {
+      params.sudoAccess = true;
+    }
     return params;
   }
 
   private void waitForNodeAgent(NodeAgent nodeAgent) {
     nodeAgentClient.waitForServerReady(nodeAgent, Duration.ofMinutes(2));
     nodeAgent.saveState(State.READY);
   }
-
-  private boolean isOnPremFullyManual(UUID universeUuid, NodeDetails nodeDetails) {
-    Universe universe = Universe.getOrBadRequest(universeUuid);
-    Cluster cluster = universe.getCluster(nodeDetails.placementUuid);
-    if (cluster.userIntent.providerType != CloudType.onprem) {
-      return false;
-    }
-    Provider provider = Provider.getOrBadRequest(UUID.fromString(cluster.userIntent.provider));
-    return provider.getDetails().skipProvisioning;
-  }
 }

managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java

@@ -2144,6 +2144,7 @@ public SubTaskGroup createInstallNodeAgentTasks(
               params.nodeAgentInstallDir = installPath;
               params.nodeAgentPort = serverPort;
               params.reinstall = reinstall;
+              params.sudoAccess = true;
               if (StringUtils.isNotEmpty(n.sshUserOverride)) {
                 params.sshUser = n.sshUserOverride;
               }

managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallNodeAgent.java

@@ -53,10 +53,11 @@ protected InstallNodeAgent(
   public static class Params extends NodeTaskParams {
     public int nodeAgentPort = DEFAULT_NODE_AGENT_PORT;
     public String nodeAgentInstallDir;
+    public String sshUser;
     public UUID customerUuid;
     public boolean reinstall;
     public boolean airgap;
-    public String sshUser;
+    public boolean sudoAccess;
   }
 
   @Override
@@ -115,10 +116,7 @@ public NodeAgent install() {
         installerFiles.getCreateDirs().stream()
             .map(dir -> dir.toString())
             .collect(Collectors.toSet());
-    // Create the staging directory with sudo first, make it writable for all users.
-    // This is done because some on-prem nodes may not have write permission to /tmp.
-    List<String> command =
-        ImmutableList.of("sudo", "mkdir", "-m", "777", "-p", stagingDir.toString());
+    List<String> command = getCommand("mkdir", "-m", "777", "-p", stagingDir.toString());
     log.info("Creating staging directory: {}", command);
     nodeUniverseManager.runCommand(node, universe, command, shellContext).processErrors();
 
@@ -177,7 +175,7 @@ public NodeAgent install() {
     if (taskParams().airgap) {
       sb.append(" --airgap");
     }
-    command = ImmutableList.of("sudo", "-H", "/bin/bash", "-c", sb.toString());
+    command = getCommand("/bin/bash", "-c", sb.toString());
     log.debug("Running node agent installation command: {}", command);
     try {
       nodeUniverseManager
@@ -188,10 +186,15 @@ public NodeAgent install() {
       throw e;
     }
     nodeAgent.saveState(State.REGISTERED);
+    sb.setLength(0);
+    sb.append("systemctl");
+    if (!taskParams().sudoAccess) {
+      sb.append(" --user");
+    }
+    sb.append(" is-active --quiet yb-node-agent");
+    command = getCommand("/bin/bash", "-c", sb.toString());
     log.debug("Waiting for node agent service to be running");
-    command =
-        ImmutableList.of(
-            "sudo", "-H", "/bin/bash", "-c", "systemctl is-active --quiet yb-node-agent");
+    log.debug("Running systemd command: {}", command);
     try {
       nodeUniverseManager
           .runCommand(node, universe, command, shellContext)
@@ -207,4 +210,12 @@ public NodeAgent install() {
   public void run() {
     install();
   }
+
+  private List<String> getCommand(String... args) {
+    ImmutableList.Builder<String> commandBuilder = ImmutableList.builder();
+    if (taskParams().sudoAccess) {
+      commandBuilder.add("sudo", "-H");
+    }
+    return commandBuilder.add(args).build();
+  }
 }