[cli] Simpler networking config explanation & command line overrides (e…

…clipse-che#4114)

* Simpler networking config explanation & command line overrides

dockerfiles/base/scripts/base/commands/cmd_config.sh

@@ -139,6 +139,17 @@ generate_configuration_with_puppet() {
     fi
   fi
 
+  for element in "${CLI_ENV_ARRAY[@]}" 
+  do
+    var1=$(echo $element | cut -f1 -d=)
+    var2=$(echo $element | cut -f2 -d=)
+
+    if [[ $var1 == CHE_* ]] ||
+       [[ $var1 == ${CHE_PRODUCT_NAME}_* ]]; then
+      WRITE_PARAMETERS+=" -e \"$var1=$var2\""
+    fi
+  done
+
   GENERATE_CONFIG_COMMAND="docker_run \
                   --env-file=\"${REFERENCE_CONTAINER_ENVIRONMENT_FILE}\" \
                   --env-file=/version/$CHE_VERSION/images \

dockerfiles/base/scripts/base/startup_02_pre_docker.sh

@@ -17,19 +17,31 @@ log() {
 }
 
 warning() {
+  if [ -z ${2+x} ]; then
+    local PRINT_COMMAND=""
+    local PRINT_STATEMENT=$1
+  else
+    local PRINT_COMMAND="($CHE_MINI_PRODUCT_NAME $1): "
+    local PRINT_STATEMENT=$2
+  fi
+
   if is_warning; then
-    printf  "${YELLOW}WARN:${NC} %s\n" "${1}"
+    printf "${YELLOW}WARN:${NC} %b%b\n" \
+              "${PRINT_COMMAND}" \
+              "${PRINT_STATEMENT}"
   fi
-  log $(printf "WARN: %s\n" "${1}")
+  log $(printf "INFO: %b %b\n" \
+        "${PRINT_COMMAND}" \
+        "${PRINT_STATEMENT}")
 }
 
 info() {
   if [ -z ${2+x} ]; then
-    PRINT_COMMAND=""
-    PRINT_STATEMENT=$1
+    local PRINT_COMMAND=""
+    local PRINT_STATEMENT=$1
   else
-    PRINT_COMMAND="($CHE_MINI_PRODUCT_NAME $1): "
-    PRINT_STATEMENT=$2
+    local PRINT_COMMAND="($CHE_MINI_PRODUCT_NAME $1): "
+    local PRINT_STATEMENT=$2
   fi
   if is_info; then
     printf "${GREEN}INFO:${NC} %b%b\n" \

dockerfiles/base/scripts/base/startup_04_pre_cli_init.sh

@@ -31,10 +31,46 @@ cli_init() {
     return 2;
   fi
 
+  CLI_ENV=$(docker inspect --format='{{.Config.Env}}' $(get_this_container_id))
+  CLI_ENV=${CLI_ENV#*[}
+  CLI_ENV=${CLI_ENV%*]}
+  IFS=' ' read -r -a CLI_ENV_ARRAY <<< "$CLI_ENV"
+
   if is_initialized; then 
     CHE_HOST_LOCAL=$(get_value_of_var_from_env_file ${CHE_PRODUCT_NAME}_HOST)
-    if [[ "${CHE_HOST}" != "${CHE_HOST_LOCAL}" ]]; then
-      warning "${CHE_PRODUCT_NAME}_HOST (${CHE_HOST}) overridden by ${CHE_ENVIRONMENT_FILE} (${CHE_HOST_LOCAL})"
+    CHE_HOST_ENV=$(get_value_of_var_from_env ${CHE_PRODUCT_NAME}_HOST)
+    if [[ "${CHE_HOST_ENV}" != "" ]] && 
+       [[ "${CHE_HOST_ENV}" != "${CHE_HOST_LOCAL}" ]]; then
+      warning "cli" "'${CHE_PRODUCT_NAME}_HOST=${CHE_HOST_ENV}' from command line overriding '${CHE_PRODUCT_NAME}_HOST=${CHE_HOST_LOCAL}' from ${CHE_ENVIRONMENT_FILE}"
+      CHE_HOST=$CHE_HOST_ENV
+    fi
+
+    if [[ "${CHE_HOST_ENV}" = "" ]] && 
+       [[ "${CHE_HOST_LOCAL}" != "" ]]; then
+      CHE_HOST=${CHE_HOST_LOCAL}
+
+      if [[ "${CHE_HOST}" != "${GLOBAL_HOST_IP}" ]]; then
+        warning "cli" "'${CHE_PRODUCT_NAME}_HOST=${CHE_HOST_LOCAL}' is != discovered IP '${GLOBAL_HOST_IP}'"
+      fi
+    fi
+
+    CHE_PORT_LOCAL=$(get_value_of_var_from_env_file ${CHE_PRODUCT_NAME}_PORT)
+    CHE_PORT_ENV=$(get_value_of_var_from_env ${CHE_PRODUCT_NAME}_PORT)
+
+    if [[ "${CHE_PORT_ENV}" != "" ]] &&
+       [[ "${CHE_PORT_LOCAL}" != "" ]] && 
+       [[ "${CHE_PORT_ENV}" != "${CHE_PORT_LOCAL}" ]]; then
+      warning "cli" "'${CHE_PRODUCT_NAME}_PORT=${CHE_PORT_ENV}' from command line overriding '${CHE_PRODUCT_NAME}_PORT=${CHE_PORT_LOCAL}' from ${CHE_ENVIRONMENT_FILE}"
+      CHE_PORT=$CHE_PORT_ENV
+    fi
+
+    if [[ "${CHE_PORT_ENV}" = "" ]] && 
+       [[ "${CHE_PORT_LOCAL}" != "" ]]; then
+      CHE_PORT=${CHE_PORT_LOCAL}
+
+      if [[ "${CHE_PORT}" != "${DEFAULT_CHE_PORT}" ]]; then
+        CHE_PORT=${CHE_PORT_LOCAL}
+      fi
     fi
   fi
 
@@ -291,6 +327,18 @@ get_value_of_var_from_env_file() {
   local LOOKUP_LOCAL=$(docker_run --env-file="${REFERENCE_CONTAINER_ENVIRONMENT_FILE}" \
                                 ${BOOTSTRAP_IMAGE_ALPINE} sh -c "echo \$$1")
   echo $LOOKUP_LOCAL
-
 }
 
+# Returns the value of variable from environment array.
+get_value_of_var_from_env() {
+  for element in "${CLI_ENV_ARRAY[@]}" 
+  do
+    var1=$(echo $element | cut -f1 -d=)
+    var2=$(echo $element | cut -f2 -d=)
+
+    if [ $var1 = $1 ]; then
+      echo $var2
+    fi
+  done
+  echo ""
+}

dockerfiles/init/manifests/che.env

@@ -11,9 +11,15 @@
 #     If you are running this on a local system, we auto-detect this value as the IP
 #     address of your Docker daemon. On many systems, especially those from cloud hosters
 #     like DigitalOcean, you must explicitly set this to the external IP address or
-#     DNS entry provided by the provider.
+#     DNS entry provided by the provider. This value can be overridden on the command
+#     line with '-e CHE_HOST=<value>'.
 #CHE_HOST=localhost
 
+# Port
+#     The port on the host Che will bind itself to. This value can be overridden on the 
+#     command line with '-e CHE_PORT=<value>'.
+#CHE_PORT=8080
+
 # Proxies
 #     Che's internal services such as Java & curl need system properties applied 
 #     so they can reach the Internet. Che uses the Internet to reach DockerHub
@@ -124,78 +130,107 @@
 #
 # Browser --> Che Server
 #    1. Default is 'http://localhost:${SERVER_PORT}/wsmaster/api'.
-#    2. Else use the value of che.api
+#    2. Else use the value of CHE_API
 #
 # Che Server --> Docker Daemon Progression:
-#    1. Use the value of che.docker.daemon_url
+#    1. Use the value of CHE_DOCKER_DAEMON__URL
 #    2. Else, use the value of DOCKER_HOST system variable
 #    3. Else, use Unix socket over unix:///var/run/docker.sock
 #
-# Che Server --> Workspace Connection (see Workspace Address Resolution Strategy, below):
+# Che Server --> Workspace Connection:
 #    - If CHE_DOCKER_SERVER__EVALUATION__STRATEGY is 'default':
-#        1. Use the value of che.docker.ip
+#        1. Use the value of CHE_DOCKER_IP
 #        2. Else, if server connects over Unix socket, then use localhost
 #        3. Else, use DOCKER_HOST
 #    - If CHE_DOCKER_SERVER__EVALUATION__STRATEGY is 'docker-local':
-#        1. Use the address of the workspace container within the docker network and exposed ports
-#        2. If address is missing, if server connects over Unix socket, then use localhost and published ports
+#        1. Use the address of the workspace container within the docker network 
+#           and exposed ports
+#        2. If address is missing, if server connects over Unix socket, then use 
+#           localhost and exposed ports
 #        3. Else, use DOCKER_HOST and published ports
 #
-# Browser --> Workspace Connection (see Workspace Address Resolution Strategy, below):
+# Browser --> Workspace Connection:
 #    - If CHE_DOCKER_SERVER__EVALUATION__STRATEGY is 'default':
-#        1. If set use the value of che.docker.ip.external
-#        2. Else if set use the value of che.docker.ip
+#        1. If set use the value of CHE_DOCKER_IP_EXTERNAL
+#        2. Else if set use the value of CHE_DOCKER_IP
 #        3. Else, if server connects over Unix socket, then use localhost
 #        4. Else, use DOCKER_HOST
 #    - If CHE_DOCKER_SERVER__EVALUATION__STRATEGY is 'docker-local':
-#        1. If set use the value of che.docker.ip.external
-#        2. Else use the address of the workspace container within the docker network, if it is set
-#        3. If address is missing, if server connects over Unix socket, then use localhost
+#        1. If set use the value of CHE_DOCKER_IP_EXTERNAL
+#        2. Else use the address of the workspace container within the docker network, 
+#           if it is set
+#        3. If address is missing, if server connects over Unix socket, then use 
+#           localhost
 #        4. Else, use DOCKER_HOST
 #
 # Workspace Agent --> Che Server
-#    1. Default is http://che-host:${SERVER_PORT}/wsmaster/api, where che-host is IP of server.
-#    2. Else, use value of che.workspace.che_server_endpoint
-#    3. Else, if 'docker0' interface is unreachable, then che-host replaced with
+#    1. Default is 'http://che-host:${SERVER_PORT}/wsmaster/api', where 'che-host' 
+#       is IP of server.
+#    2. Else, use value of CHE_WORKSPACE_CHE__SERVER__ENDPOINT
+#    3. Else, if 'docker0' interface is unreachable, then 'che-host' replaced with
 #       172.17.42.1 or 192.168.99.1
 #    4. Else, print connection exception
 
+# Che Server API Endpoint
+#     The location of the API end point where dashboard and IDE clients will look for
+#     interacting with the Che server, which we also call the workspace master.
+#CHE_API=http://localhost:${SERVER_PORT}/wsmaster/api
 
+# Docker Daemon URL
+#     How the Che server will discover the location of the Docker daemon. If this is
+#     not set, then Che will use DOCKER_HOST or the default unix:///var/run/docker.sock.
+#     It would be rare to need to set this as most clients set DOCKER_HOST or volume
+#     mount a Docker socket when starting the Che CLI.
+#CHE_DOCKER_DAEMON__URL=NULL
 
-########################################################################################
-#####                                                                              #####
-#####                                     DOCKER                                   #####
-#####                                                                              #####
-#
-# IP Address
-#     The IP address of the Docker daemon that is running on your host. We do a
-#     self-discovery to auto-set this. You can combine this with DOCKER_HOST to change
-#     communications from socket to TCP. This value will be set to DOCKER_MACHINE_HOST
-#     env variable for the Che Docker container when it boots - it's how we determine
-#     what containers will boot.
-#CHE_DOCKER_IP=172.17.0.1
-
-# External Host Name
-#     The hostname that a browser should use to connect to a workspace container.
-#     Only set this if your workspace containers are not directly pingable. This is unusual,
-#     but happens for example in Docker for Mac when containers are in a VM.
-#CHE_DOCKER_IP_EXTERNAL=NULL
+# Docker Host
+#     How Che will connect to the Docker host if CHE_DOCKER_DAEMON__URL is not set. This
+#     value can be provided on the command line as well. If this value is not set, then
+#     Che will use the volume mount of unix:///var/run/docker.sock.
+#DOCKER_HOST=tcp://localhost:2375
 
 # Workspace Address Resolution Strategy
-#     The strategy to be used to determine servers exposed by workspaces.
+#     The strategy to determine how workspace servers users launch will be exposed. 
+#     If the Che server and your workspaces are on different networks that flow through a
+#     firewall, those firewalls can prevent communications between Che and workspaces.
+#     The default strategy exposes workspaces with the IP of Docker and ephemeral ports.
+#     A Docker local strategy uses internal Docker network IP addresses and exposed ports.
+#
 #     Options:
-#       - 'default':      internal address is address of docker host and ephemeral port are used
-#       - 'docker-local': internal address is address of container within docker network, and exposed ports
-#                         are used.
-#     The 'docker-local' strategy may be useful if a firewall prevents communication between che-server and
-#     workspace containers, but will prevent communication when che-server and workspace containers are not
+#       - 'default':      internal address is DOCKER_HOST with ephemeral ports
+#       - 'docker-local': internal address is Docker network container address with
+#                         exposed ports (not ephemeral)
 #     on the same Docker network.
 #CHE_DOCKER_SERVER__EVALUATION__STRATEGY=docker-local
 
-# Docker Host
-#     How Che will connect to the Docker Host.
-#DOCKER_HOST=tcp://localhost:2375
+# Docker IP Address
+#     The IP address of the Docker daemon that is running on your host. The Che server
+#     uses this IP address to establish a connection to your workspaces when they
+#     first boot. This is different from how the Che server first establishes a
+#     connection to the Docker daemon. After a Che server has instructed a Docker
+#     Daemon to create a workspace, the Che server then discovers the location of the
+#     workspace using the value of this IP address. We use auto-discovery to determine
+#     this IP address. You can provide this value to override the default discovery
+#     which searches Unix sockets and DOCKER_HOST. 
+#CHE_DOCKER_IP=172.17.0.1
+
+# External Docker IP Address
+#     The external IP address used by browsers to connect to workspace containers that
+#     are running on a Docker network. After Che has created a workspace runtime, those
+#     Docker containers need to be reachable by remote browsers. Browsers will default
+#     to using CHE_DOCKER_IP, then unix sockets, or DOCKER_HOST if this is not set. 
+#     This will generally work if the browser and the workspace are on the same network.
+#     However, if the browser and the workspaces are on different networks, then you
+#     may need to set this value to the external IP address of your host or Docker to
+#     make workspaces reachable.
+#CHE_DOCKER_IP_EXTERNAL=NULL
+
 
+########################################################################################
+#####                                                                              #####
+#####                                     DOCKER                                   #####
+#####                                                                              #####
+#
 # Docker Registry for Workspace Snapshots
 #     Docker is the default machine implementation within Che. Workspaces are powered
 #     by machines that are constructed when the workspace is started. The images used to
@@ -218,7 +253,6 @@
 #CHE_WORKSPACE_AUTO__SNAPSHOT=false
 #CHE_WORKSPACE_AUTO__RESTORE=false
 
-
 # Private Docker Images
 #     If pushing snap images to a registry requires authenticated access to the
 #     registry. Or, if your stacks reference private images which require authenticated

dockerfiles/init/modules/che/templates/che.env.erb

@@ -60,7 +60,7 @@ JAVA_HTTPS_PROXY_PORT=-Dhttps.proxyPort=<%= @che_https_proxy.gsub(/^https?\:\/\/
 <% if ! @che_no_proxy.empty? -%>
 JAVA_NO_PROXY=-Dhttp.nonProxyHosts='<%= @che_no_proxy.gsub(/^https?\:\/\//, '').gsub(/^www./,'').gsub(',','|') %>|'
 <% end -%>
-JAVA_OPTS=-Xms512m -server -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:+ScavengeBeforeFullGC -XX:+CMSScavengeBeforeRemark <% if ! @che_http_proxy.empty? or ! @che_https_proxy.empty? -%>$JAVA_HTTP_PROXY_SET<% end -%> <% if ! @che_http_proxy.empty? -%>$JAVA_HTTP_PROXY_HOST $JAVA_HTTP_PROXY_PORT<% end -%> <% if ! @che_https_proxy.empty? -%>$JAVA_HTTPS_PROXY_HOST $JAVA_HTTPS_PROXY_PORT<% end -%><%- if ! @che_no_proxy.empty? -%> $JAVA_NO_PROXY<% end -%><% if @che_http_proxy.include? '@' -%> $JAVA_HTTP_USER_NAME $JAVA_HTTP_USER_PASSWORD<% end -%><% if @che_https_proxy.include? '@' -%> $JAVA_HTTPS_USER_NAME $JAVA_HTTPS_USER_PASSWORD<% end %>
+JAVA_OPTS=-Xms512m -Djava.security.egd=file:/dev/./urandom -server -XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:+ScavengeBeforeFullGC -XX:+CMSScavengeBeforeRemark <% if ! @che_http_proxy.empty? or ! @che_https_proxy.empty? -%>$JAVA_HTTP_PROXY_SET<% end -%> <% if ! @che_http_proxy.empty? -%>$JAVA_HTTP_PROXY_HOST $JAVA_HTTP_PROXY_PORT<% end -%> <% if ! @che_https_proxy.empty? -%>$JAVA_HTTPS_PROXY_HOST $JAVA_HTTPS_PROXY_PORT<% end -%><%- if ! @che_no_proxy.empty? -%> $JAVA_NO_PROXY<% end -%><% if @che_http_proxy.include? '@' -%> $JAVA_HTTP_USER_NAME $JAVA_HTTP_USER_PASSWORD<% end -%><% if @che_https_proxy.include? '@' -%> $JAVA_HTTPS_USER_NAME $JAVA_HTTPS_USER_PASSWORD<% end %>
 
 # java opts for ws agent
 CHE_WORKSPACE_JAVA_OPTIONS=-Xms256m -Xmx2048m -Djava.security.egd=file:/dev/./urandom <% if ! @http_proxy_for_che_workspaces.empty? or ! @https_proxy_for_che_workspaces.empty? -%>-Dhttp.proxySet=true<% end -%><% if ! @http_proxy_for_che_workspaces.empty? -%><% if ! @http_proxy_for_che_workspaces.empty? and @http_proxy_for_che_workspaces.include? '@' -%> -Dhttp.proxyUser=<%= @http_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[0].split(':')[0] %> -Dhttp.proxyPassword=<%= @http_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[0].split(':')[1] %> -Dhttp.proxyHost=<%= @http_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[1].split(':')[0] %> -Dhttp.proxyPort=<%= @http_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[1].split(':')[1].gsub(/\/.*/,'') %><% else -%> -Dhttp.proxyHost=<%= @http_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split(':')[0] %> -Dhttp.proxyPort=<%= @http_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split(':')[1].gsub(/\/.*/,'') %><% end -%><% end -%><% if ! @https_proxy_for_che_workspaces.empty? -%><% if @https_proxy_for_che_workspaces.include? '@' -%> -Dhttps.proxyUser=<%= @https_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[0].split(':')[0] %> -Dhttps.proxyPassword=<%= @https_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[0].split(':')[1] %> -Dhttps.proxyHost=<%= @https_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[1].split(':')[0] %> -Dhttps.proxyPort=<%= @https_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split('@')[1].split(':')[1].gsub(/\/.*/,'') %><% else -%> -Dhttps.proxyHost=<%= @https_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split(':')[0] %> -Dhttps.proxyPort=<%= @https_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').split(':')[1].gsub(/\/.*/,'') %><% end -%><% end -%><% if ! @che_no_proxy.empty? -%> -Dhttp.nonProxyHosts='<%= @no_proxy_for_che_workspaces.gsub(/^https?\:\/\//, '').gsub(/^www./,'').gsub(',','|') %>|'<% end -%>