Template bootstrap script

.dockerignore

@@ -12,3 +12,9 @@ _build/*
 **/*.beam
 #no erlcinfo files (in order to have cleaner logs when compiling in the container)
 **/erlcinfo
+# no vim temp files
+**/.*.sw*
+.git
+
+tools/pkg/Dockerfile_rpm
+tools/pkg/packages/

.gitignore

@@ -127,3 +127,6 @@ codecov.json
 
 auto_small_tests.spec
 big_tests/auto_big_tests.spec
+*.rpm
+
+tools/pkg/packages/

doc/user-guide/Bootstrap-Scripts.md

@@ -18,3 +18,58 @@ Environment variables, available from scripts:
 
 - `ERTS_PATH` - path to Erlang Runtime System, used by MongooseIM.
 - `MIM_DIR` - MongooseIM release installation directory.
+
+
+# Templating bootstrap script
+
+The script `bootstrap20-template.escript` renders files from the `templates/` directory and writes
+result files into the `etc/` directory. If you need the result files in a separate directory,
+create another script `bootstrap30-template.sh`, that moves files into a proper location.
+
+
+The `etc/templates.ini` file contains default template variables.
+
+
+A template config example:
+
+```erlang
+[options]
+demo_session_lifetime = 600
+demo_tls_versions = 'tlsv1.2', 'tlsv1.3'
+```
+
+Only lowercase variables are allowed in `templates.ini`.
+
+You can redeclare options using environment variables when executing the bootstrap script:
+
+```bash
+MIM_DEMO_SESSION_LIFETIME=700 mongooseimctl bootstrap
+```
+
+Environment variables should have a `MIM_` prefix. The variable names are case-insensitive
+(but we suggest to use the uppercase variable names for consistency).
+
+# Demo template
+
+A demo template is located in `rel/files/templates/demo.config`.
+It is copied into the `/templates` directory inside your release directory.
+
+
+
+
+# Testing templating scripts
+
+Templating script source code: `rel/files/scripts/bootstrap20-template.escript`.
+
+Testing script code:
+
+```bash
+tools/pkg/scripts/smoke_test.sh
+tools/pkg/scripts/smoke_templates.escript
+```
+
+Testing command:
+
+```bash
+PRESET=pkg pkg_PLATFORM=centos_7 ESL_ERLANG_PKG_VER=22.1.8-2 ./tools/travis-test.sh
+```

rebar.config

@@ -105,7 +105,11 @@
                    {copy, "tools/ssl/ca/cacert.pem",        "priv/ssl/cacert.pem"},
 
                    {copy,     "rel/files/erl",          "erts-\{\{erts_vsn\}\}/bin/erl"},
-                   {copy, "rel/files/scripts",   "scripts"},
+                   %% Copy the whole directory scripts into scripts.
+                   %% Still works, if the destination "scripts/" directory exists.
+                   {copy, "rel/files/scripts",   "./"},
+                   {copy, "rel/files/templates", "./"},
+                   {copy, "rel/files/templates.ini", "etc/templates.ini"},
 
                    {template, "rel/files/nodetool",     "erts-\{\{erts_vsn\}\}/bin/nodetool"},
 

rel/files/mongooseimctl

@@ -6,9 +6,21 @@ RUNNER_BASE_DIR="${RUNNER_SCRIPT_DIR%/*}"
 RUNNER_ETC_DIR="{{mongooseim_etc_dir}}"
 RUNNER_USER="{{mongooseim_runner_user}}"
 
+# Return variables with MIM_ prefix
+function preserved_variables
+{
+    EXP=$(export -p | grep '^declare -x MIM_')
+    # if not empty
+    if [ ! -z "$EXP" ]; then
+        # concat lines with " && " separator and add the separator at the end
+        echo "${EXP//$'\n'/ && } && "
+    fi
+}
+
 # Make sure this script is running as the appropriate user
 if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then
-    exec runuser -l "$RUNNER_USER" -c "$0 $*"
+    # Preserve some env variables
+    exec runuser -s /bin/bash -l "$RUNNER_USER" -c "$(preserved_variables) $0 $*"
 fi
 
 MIM_DIR="${RUNNER_SCRIPT_DIR%/*}"
@@ -115,6 +127,14 @@ print_install_dir ()
     echo "$MIM_DIR"
 }
 
+run_escript()
+{
+    export MIM_DIR="$MIM_DIR"
+
+    shift
+    "$ERTS_PATH"/escript $@
+}
+
 start ()
 {
     "$RUNNER_SCRIPT_DIR"/mongooseim start
@@ -162,6 +182,7 @@ help ()
     echo "Extra Commands:"
     echo "  bootstrap           Executes MongooseIM init scripts (used for initial configuration)"
     echo "  print_install_dir   Prints path to MongooseIM release directory"
+    echo "  escript             Runs escript command using embedded Erlang Runtime System"
     echo ""
 }
 
@@ -381,5 +402,6 @@ case $1 in
     'started') wait_for_status_file started 60 1 || true; wait_for_status 0 30 2;; # wait 30x2s before timeout
     'stopped') is_started_status_file && wait_for_status_file stopped 30 1 || true; wait_for_status 3 15 2; stop_epmd;; # wait 15x2s before timeout
     'print_install_dir') print_install_dir;;
+    'escript') run_escript $@;;
     *) ctl $QUOTED_ARGS;;
 esac

rel/files/scripts/bootstrap20-template.escript

@@ -0,0 +1,113 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -noinput
+
+-mode(compile).
+-include_lib("kernel/include/file.hrl").
+
+main(_) ->
+    io:format("Template script started~n", []),
+    MIM_DIR = os:getenv("MIM_DIR"),
+    TemplateConfigPath = filename:join(MIM_DIR, "etc/templates.ini"),
+    case file:read_file(TemplateConfigPath) of
+        {ok, TemplateConfigBin} ->
+            init_and_template(MIM_DIR, TemplateConfigPath, TemplateConfigBin);
+        _ ->
+            io:format("Skip templating because config ~p does not exist",
+                      [TemplateConfigPath])
+    end.
+
+init_and_template(MIM_DIR, TemplateConfigPath, TemplateConfigBin) ->
+    check_mim_dir(MIM_DIR),
+    import_libraries(MIM_DIR),
+    ensure_deps(),
+    TemplateConfig = parse_template_config(TemplateConfigPath, TemplateConfigBin),
+
+    %% Options defined in the ini config file
+    FileOpts = maps:from_list(proplists:get_value(options, TemplateConfig, [])),
+
+    LowerEnvVars = [{string:to_lower(K), V} || {K, V} <- os:list_env_vars()],
+
+    %% Add all env variables with prefix MIM_
+    EnvVars = maps:from_list([{list_to_atom(K), list_to_binary(V)}
+                              || {"mim_" ++ K, V} <- LowerEnvVars]),
+
+    io:format("Found ~p env variables~n", [maps:size(EnvVars)]),
+
+    %% EnvVars have higher priority
+    Opts = maps:merge(FileOpts, EnvVars),
+
+    %% Dump variables into a file for debugging
+    VarsPath = filename:join(MIM_DIR, "etc/final_template_vars.config"),
+    file:write_file(VarsPath, io_lib:format("~p.", [Opts])),
+
+    InputDir = filename:join(MIM_DIR, "templates"),
+    OutputDir = filename:join(MIM_DIR, "etc"),
+
+    InputFiles = list_dir_safe(InputDir),
+    io:format("InputFiles ~p~n", [InputFiles]),
+    [render_file(filename:join(InputDir, File), filename:join(OutputDir, File), Opts)
+     || File <- InputFiles],
+
+    ok.
+
+list_dir_safe(Dir) ->
+    case file:list_dir(Dir) of
+        {ok, Files} ->
+            Files;
+        Other ->
+            io:format("Failed to list directory ~p~n Reason ~p~n", [Dir, Other]),
+            []
+    end.
+
+render_file(Src, Dst, Params) ->
+    {ok, #file_info{mode = Mode}} = file:read_file_info(Src),
+    {ok, Bin} = file:read_file(Src),
+    Out = bbmustache:render(Bin, Params, mustache_opts()),
+    ok = filelib:ensure_dir(Dst),
+    Result = file:write_file(Dst, Out),
+    io:format("Result file written ~p: ~p~n", [Dst, Result]),
+    ok = file:change_mode(Dst, Mode).
+
+mustache_opts() ->
+    [{escape_fun, fun id/1}, {key_type, atom}, {value_serializer, fun id/1}].
+
+id(X) -> X.
+
+check_mim_dir(false) ->
+    halt_with_error("MIM_DIR is not set");
+check_mim_dir(MIM_DIR) ->
+    io:format("MIM_DIR is set to ~p~n", [MIM_DIR]),
+    ok.
+
+halt_with_error(Reason) ->
+    io:format("ERROR ~ts", [Reason]),
+    halt(1).
+
+import_libraries(MIM_DIR) ->
+    Pattern = filename:join(MIM_DIR, "lib/*/ebin"),
+    Paths = filelib:wildcard(Pattern),
+    code:add_paths(Paths).
+
+ensure_deps() ->
+    ensure_dep(eini),
+    ensure_dep(bbmustache),
+    ok.
+
+ensure_dep(Dep) ->
+    case application:ensure_all_started(Dep) of
+        {ok, _} ->
+            ok;
+        Other ->
+            io:format("ensure_all_started for ~p returns ~p~n", [Dep, Other]),
+            halt_with_error("ensure_dep_failed")
+    end.
+
+parse_template_config(TemplateConfigPath, TemplateConfigBin) ->
+    case eini:parse(TemplateConfigBin) of
+        {ok, Config} ->
+            Config;
+        Other ->
+            io:format("Failed to parse ~p~n Reason ~p~n", [TemplateConfigPath, Other]),
+            halt_with_error("parse_template_config_failed")
+    end.

rel/files/templates.ini

@@ -0,0 +1,3 @@
+[options]
+demo_session_lifetime = 600
+demo_tls_versions = 'tlsv1.2', 'tlsv1.3'

rel/files/templates/demo.config

@@ -0,0 +1,7 @@
+%% Template to show how the bootstrap template script works and for testing
+[
+   {ssl, [
+        {session_lifetime, {{{ demo_session_lifetime }}} },
+        {protocol_version, [ {{{ demo_tls_versions }}} ]}
+       ]}
+].

tools/configure

@@ -51,7 +51,8 @@ app_opts() ->
      {"cassandra",     ["cqerl"],           include_driver("cassandra")},
      {"elasticsearch", ["tirerl"],          include_driver("elasticsearch")},
      {"aws",           ["erlcloud"],        "include support for AWS services"},
-     {"amqp_client",   ["amqp_client"],     "include AMQP client"}].
+     {"amqp_client",   ["amqp_client"],     "include AMQP client"},
+     {"templates",     ["eini", "bbmustache"], "include applications for templating"}].
 
 opts() ->
     [{"prefix",  (#opts{})#opts.prefix,  "Installation PREFIX directory"},

tools/pkg/Dockerfile_deb

@@ -44,6 +44,7 @@ RUN apt-get update; dpkg -i *.deb; apt-get install -y -f
 # Simple check if MiM works
 COPY --from=builder /root/mongooseim/tools/wait-for-it.sh .
 COPY --from=builder /root/mongooseim/tools/pkg/scripts/smoke_test.sh .
+COPY --from=builder /root/mongooseim/tools/pkg/scripts/smoke_templates.escript .
 
 RUN ./smoke_test.sh
 

tools/pkg/Dockerfile_rpm

@@ -1,6 +1,15 @@
 # vi: ft=dockerfile
 ARG dockerfile_platform
 
+# Source container is useful for preparing code for the actual build.
+FROM $dockerfile_platform AS source
+COPY . /SOURCE
+# Separate the main code from the testing code.
+# So that any changes in the testing code do not trigger cache invalidation in the builder.
+RUN mkdir /TESTS \
+    && mv /SOURCE/tools/pkg/scripts/smoke_test.sh /TESTS/ \
+    && mv /SOURCE/tools/pkg/scripts/smoke_templates.escript /TESTS/
+
 FROM $dockerfile_platform AS builder
 
 # Install the build dependencies
@@ -23,7 +32,7 @@ RUN curl -O https://packages.erlang-solutions.com/erlang-solutions-2.0-1.noarch.
 # including not commited changes are used to build the package
 RUN rpmdev-setuptree
 WORKDIR /root/rpmbuild
-COPY . ./BUILD/mongooseim
+COPY --from=source /SOURCE ./BUILD/mongooseim
 
 RUN cp ./BUILD/mongooseim/tools/pkg/scripts/rpm/mongooseim.spec ./SPECS/.
 RUN cp ./BUILD/mongooseim/tools/pkg/scripts/rpm/mongooseim.service \
@@ -44,7 +53,7 @@ RUN yum -y update; yum install -y mongooseim*.rpm
 
 # Simple check if MiM works
 COPY --from=builder /root/rpmbuild/BUILD/mongooseim/tools/wait-for-it.sh .
-COPY --from=builder /root/rpmbuild/BUILD/mongooseim/tools/pkg/scripts/smoke_test.sh .
+COPY --from=source /TESTS/* ./
 
 RUN ./smoke_test.sh
 

tools/pkg/scripts/rpm/mongooseim.spec

@@ -70,6 +70,7 @@ rm -rf %{buildroot}
 
 %config	 %{_sysconfdir}/mongooseim/*
 
+%attr(770,mongooseim,mongooseim) %{_prefix}/lib/mongooseim/etc
 %attr(770,mongooseim,mongooseim) %dir %{_localstatedir}/lib/mongooseim
 %attr(770,mongooseim,mongooseim) %dir %{_localstatedir}/log/mongooseim
 %attr(770,mongooseim,mongooseim) %dir %{_localstatedir}/lock/mongooseim

tools/pkg/scripts/smoke_templates.escript

@@ -0,0 +1,15 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -noinput
+
+-mode(compile).
+
+main(_) ->
+    io:format("Template smoke script started~n", []),
+    MIM_DIR = os:getenv("MIM_DIR"),
+    DemoPath = filename:join(MIM_DIR, "etc/demo.config"),
+    {ok, Cfg} = file:consult(DemoPath),
+    [[{ssl,SSL}]] = Cfg,
+    ['tlsv1.2','tlsv1.3'] = proplists:get_value(protocol_version, SSL),
+    700 = proplists:get_value(session_lifetime, SSL),
+    ok.

tools/pkg/scripts/smoke_test.sh

@@ -5,18 +5,30 @@
 set -eu pipefail
 IFS=$'\n\t'
 
+echo "Check that print_install_dir works"
+MIM_DIR=$(mongooseimctl print_install_dir)
+test -d "$MIM_DIR"
+
 echo "Executing init scripts via 'mongooseimctl bootstrap'"
-# Fails, if exit code is wrong
+# Fails, if the exit code is wrong
 mongooseimctl bootstrap
 
-echo "Check, that bootstrap01-hello.sh script is executed"
+echo "Check that bootstrap01-hello.sh script is executed"
 BOOTSTRAP_RESULT=$(mongooseimctl bootstrap)
 echo "$BOOTSTRAP_RESULT" | grep "Hello from"
 
-echo "Check, that print_install_dir works"
-MIM_DIR=$(mongooseimctl print_install_dir)
-test -d "$MIM_DIR"
+# Script should be accessable by the "mongooseim" user
+mv smoke_templates.escript "$MIM_DIR/"
+
+echo "Check, that templates are correctly processed"
+echo "Override default demo_session_lifetime=600 with 700"
+# We check escaping with MIM_unused_var
+MIM_unused_var="'\n\t\t\"" MIM_demo_session_lifetime=700 mongooseimctl bootstrap
+mongooseimctl escript "$MIM_DIR/smoke_templates.escript"
 
+# Uppercase variables also work
+MIM_DEMO_SESSION_LIFETIME=700 mongooseimctl bootstrap
+mongooseimctl escript "$MIM_DIR/smoke_templates.escript"
 
 echo "Check, that bootstrap fails, if permissions are wrong"
 GOOD_SCRIPT="$MIM_DIR/scripts/bootstrap01-hello.sh"