Merge pull request #3401 from esl/mu-dyn-conf

MIM-1339 CircleCI on docker with DB backends

Changes:
* Pass files into container using env vars (instead of volumes)
* Convert previous preset jobs to be run using Docker Executors
* Cache compiled deps (and handle asn compilation case correctly)
* Print more info when testing sasl_external_suite
* Fix case when there are more than 30 comments in publish-github-comment.
* Use 2 weeks valid cache key for certs

Makefile

@@ -61,9 +61,18 @@ $(DEVNODES): certs configure.out rel/vars-toml.config
 	(. ./configure.out && \
 	DEVNODE=true $(RUN) $(REBAR) as $@ release)
 
-certs:
-	if ! openssl x509 -checkend 3600 -noout -in tools/ssl/ca/cacert.pem ; then \
-		cd tools/ssl && make clean_certs && $(MAKE); \
+maybe_clean_certs:
+	if [ "$$SKIP_CERT_BUILD" = 1 ]; then \
+		if ! openssl x509 -checkend 3600 -noout -in tools/ssl/ca/cacert.pem ; then \
+			cd tools/ssl && make clean_certs; \
+		fi \
+	fi
+
+certs: maybe_clean_certs
+	if [ "$$SKIP_CERT_BUILD" = 1 ]; then \
+		echo "Skip cert build"; \
+		else \
+		cd tools/ssl && make; \
 	fi
 
 xeplist:

big_tests/test.config

@@ -385,6 +385,7 @@
 [outgoing_pools.cassandra.default]
   scope = \"global\"
   workers = 20
+  connection.servers = [{ip_address = \"localhost\", port = 9142}]
   connection.tls.cacertfile = \"priv/ssl/cacert.pem\"
   connection.tls.verify_peer = true
 [outgoing_pools.elastic.default]

big_tests/tests/sasl_external_SUITE.erl

@@ -382,6 +382,7 @@ generate_cert(C, #{cn := User} = CertSpec) ->
     TemplateValues = prepare_template_values(User, XMPPAddrs),
     OpenSSLConfig = bbmustache:render(Template, TemplateValues),
     UserConfig = filename:join(?config(priv_dir, C), User ++ ".cfg"),
+    ct:log("OpenSSL config: ~ts~n~ts", [UserConfig, OpenSSLConfig]),
     file:write_file(UserConfig, OpenSSLConfig),
     UserKey = filename:join(?config(priv_dir, C), User ++ "_key.pem"),
 
@@ -396,22 +397,23 @@ generate_ca_signed_cert(C, User, UserConfig, UserKey ) ->
     UserCsr = filename:join(?config(priv_dir, C), User ++ ".csr"),
     Cmd = ["openssl req -config ", UserConfig, " -newkey rsa:2048 -sha256 -nodes -out ",
            UserCsr, " -keyout ", UserKey, " -outform PEM"],
-    _Out = os:cmd(Cmd),
+    Out = os:cmd(Cmd),
+    ct:log("generate_ca_signed_cert 1:~nCmd ~p~nOut ~ts", [Cmd, Out]),
     UserCert = filename:join(?config(priv_dir, C), User ++ "_cert.pem"),
     SignCmd = filename:join(?config(mim_data_dir, C), "sign_cert.sh"),
     Cmd2 = [SignCmd, " --req ", UserCsr, " --out ", UserCert],
-    LogFile = filename:join(?config(priv_dir, C), User ++ "signing.log"),
     SSLDir = filename:join([path_helper:repo_dir(C), "tools", "ssl"]),
     OutLog = os:cmd("cd " ++ SSLDir ++ " && " ++ Cmd2),
-    [] = os:cmd("echo \"" ++ OutLog ++ "\" > " ++ LogFile),
+    ct:log("generate_ca_signed_cert 2:~nCmd ~p~nOut ~ts", [Cmd2, OutLog]),
     #{key => UserKey,
       cert => UserCert}.
 
 generate_self_signed_cert(C, User, UserConfig, UserKey) ->
     UserCert = filename:join(?config(priv_dir, C), User ++ "_self_signed_cert.pem"),
     Cmd = ["openssl req -config ", UserConfig, " -newkey rsa:2048 -sha256 -nodes -out ",
            UserCert, " -keyout ", UserKey, " -x509 -outform PEM -extensions client_req_extensions"],
-    _ = os:cmd(Cmd),
+    OutLog = os:cmd(Cmd),
+    ct:log("generate_self_signed_cert:~nCmd ~p~nOut ~ts", [Cmd, OutLog]),
     #{key => UserKey,
       cert => UserCert}.
 

rebar.config

@@ -125,7 +125,7 @@
                    {copy, "tools/ssl/mongooseim/key.pem",         "priv/ssl/fake_key.pem"},
                    {copy, "tools/ssl/mongooseim/server.pem",      "priv/ssl/fake_server.pem"},
                    {copy, "tools/ssl/mongooseim/dh_server.pem",   "priv/ssl/fake_dh_server.pem"},
-                   {copy, "tools/ssl/ca/cacert.pem",        "priv/ssl/cacert.pem"},
+                   {copy, "tools/ssl/ca/cacert.pem",              "priv/ssl/cacert.pem"},
 
                    {copy,     "rel/files/erl",          "erts-\{\{erts_vsn\}\}/bin/erl"},
                    %% Copy the whole directory scripts into scripts.

rel/mim3.vars-toml.config

@@ -8,7 +8,7 @@
 {https_port, 5290}.
 {http_api_old_endpoint_port, 5292}.
 {http_api_endpoint_port, 8092}.
-{http_api_client_endpoint_port, 8093}.
+{http_api_client_endpoint_port, 8193}.
 
 {hosts, "\"localhost\", \"anonymous.localhost\", \"localhost.bis\""}.
 {default_server_domain, "\"localhost\""}.

src/cert_utils.erl

@@ -107,8 +107,7 @@ get_lserver_from_addr(V, UTF8) when is_binary(V); is_list(V) ->
 get_lserver_from_addr(_, _) -> [].
 
 
-log_exception(Cert, Class, Exception, StackTrace) ->
-    ?LOG_DEBUG(#{what => <<"cert_parsing_failed">>,
+log_exception(_Cert, Class, Exception, StackTrace) ->
+    ?LOG_ERROR(#{what => <<"cert_parsing_failed">>,
                  text => <<"failed to parse certificate">>,
-                 cert => Cert,
                  class => Class, reason => Exception, stacktrace => StackTrace}).

src/sasl/cyrsasl_external_verification.erl

@@ -34,21 +34,28 @@
 -export([verify_creds/1]).
 
 -include_lib("jid/include/jid.hrl").
+-include("mongoose_logger.hrl").
 
 -spec verify_creds(Creds :: mongoose_credentials:t()) ->
     {ok, Username :: binary()} | {error, Error :: binary()}.
 
 verify_creds(Creds) ->
     AuthId = mongoose_credentials:get(Creds, auth_id, undefined),
-    XmppAddr = case mongoose_credentials:get(Creds, xmpp_addresses) of
+    Addrs = mongoose_credentials:get(Creds, xmpp_addresses),
+    XmppAddr = case Addrs of
                    [Addr] -> Addr;
                    _ -> undefined
                end,
     CN = mongoose_credentials:get(Creds, common_name, undefined),
-    [JID | _] = [Name || Name <- [AuthId, XmppAddr, CN, <<"">>], Name =/= undefined],
+    Sources = [AuthId, XmppAddr, CN, <<"">>],
+    [JID | _] = [Name || Name <- Sources, Name =/= undefined],
     Server = mongoose_credentials:lserver(Creds),
     case jid:from_binary(JID) of
         #jid{luser = User, lserver = Server, lresource = <<"">>} when User =/= <<"">> ->
             {ok, User};
-        _ -> {error, <<"not-authorized">>}
+        Result ->
+            ?LOG_ERROR(#{what => cyrsasl_external_verification_failed,
+                         xmpp_addresses => Addrs, server => Server,
+                         auth_sources => Sources, result => Result}),
+            {error, <<"not-authorized">>}
     end.

tools/build-deps.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Hide our code from rebar to not compile it at this stage
+mv src src_old
+mv asn1 asn1_old
+./rebar3 compile
+# rebar3 could create src directory on its own
+rm -rf src asn1 asngen
+mv src_old src
+mv asn1_old asn1

tools/build-test-deps.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd big_tests
+mv src src_old
+mv tests tests_old
+../rebar3 compile
+rm -rf src tests
+mv src_old src
+mv tests_old tests

tools/certs-version.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Simple hash for our certs
+md5sum tools/ssl/mongooseim/key.pem | cut -d " " -f1

tools/circle-generate-config.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+OUT_FILE="$1"
+
+echo | base32 -w0 > /dev/null 2>&1
+if [ $? -eq 0 ]; then
+      # GNU coreutils base32, '-w' supported
+      ENCODER="base32 -w0"
+    else
+      # Openssl base32, no wrapping by default
+      ENCODER="base32"
+fi
+
+set -e
+source tools/db-versions.sh
+
+function cat32 {
+    cat "$1" | $ENCODER
+}
+
+MYSQL_CNF=$(cat32 tools/db_configs/mysql/mysql.cnf)
+MYSQL_SQL=$(cat32 priv/mysql.sql)
+MYSQL_SETUP=$(cat32 tools/docker-setup-mysql.sh)
+
+PGSQL_CNF=$(cat32 tools/db_configs/pgsql/postgresql.conf)
+PGSQL_SQL=$(cat32 priv/pg.sql)
+PGSQL_HBA=$(cat32 tools/db_configs/pgsql/pg_hba.conf)
+PGSQL_SETUP=$(cat32 tools/docker-setup-postgres.sh)
+
+MSSQL_SQL=$(cat32 priv/mssql2012.sql)
+MSSQL_SETUP=$(cat32 tools/docker-setup-mssql.sh)
+
+LDAP_SCHEMA=$(cat32 tools/db_configs/ldap/init_entries.ldif)
+LDAP_SETUP=$(cat32 tools/db_configs/ldap/init_script.sh)
+
+RIAK_SSL_CFG=$(cat32 tools/db_configs/riak/riak.conf.ssl)
+RIAK_ADV_CFG=$(cat32 tools/db_configs/riak/advanced.config)
+RIAK_SETUP=$(cat32 tools/setup_riak.escript)
+RIAK_MAM_SEARCH_SCHEMA=$(cat32 tools/mam_search_schema.xml)
+RIAK_VCARD_SEARCH_SCHEMA=$(cat32 tools/vcard_search_schema.xml)
+RIAK_SETUP_SH=$(cat32 tools/db_configs/riak/setup-riak.sh)
+
+CASSA_PROXY_CNF=$(cat32 tools/db_configs/cassandra/proxy/zazkia-routes.json)
+CASSA_ENTRY=$(cat32 tools/db_configs/cassandra/docker_entry.sh)
+CASSA_MIM_CQL_ENTRY=$(cat32 priv/cassandra.cql)
+CASSA_TEST_CQL_ENTRY=$(cat32 big_tests/tests/mongoose_cassandra_SUITE_data/schema.cql)
+
+MIM_CERT=$(cat32 tools/ssl/mongooseim/cert.pem)
+MIM_KEY=$(cat32 tools/ssl/mongooseim/key.pem)
+MIM_PRIV_KEY=$(cat32 tools/ssl/mongooseim/privkey.pem)
+MIM_DHSERVER=$(cat32 tools/ssl/mongooseim/dh_server.pem)
+INJECT_FILES=$(cat32 tools/inject-files.sh)
+CACERT=$(cat32 tools/ssl/ca/cacert.pem)
+
+PYTHON2_BASE32_DEC="python2 -c 'import base64; import sys; sys.stdout.write(base64.b32decode(sys.stdin.readline().strip()))'"
+PYTHON3_BASE32_DEC="python3 -c 'import base64; import sys; sys.stdout.buffer.write(base64.b32decode(sys.stdin.readline().strip()))'"
+
+CERTS_CACHE_KEY=$(cat certs_cache_key)
+
+sed -e "s/__MYSQL_CNF__/${MYSQL_CNF}/" \
+    -e "s/__MYSQL_SQL__/${MYSQL_SQL}/" \
+    -e "s/__MYSQL_SETUP__/${MYSQL_SETUP}/" \
+    -e "s/__MYSQL_VERSION__/${MYSQL_VERSION}/" \
+    -e "s/__PGSQL_CNF__/${PGSQL_CNF}/" \
+    -e "s/__PGSQL_SQL__/${PGSQL_SQL}/" \
+    -e "s/__PGSQL_HBA__/${PGSQL_HBA}/" \
+    -e "s/__PGSQL_SETUP__/${PGSQL_SETUP}/" \
+    -e "s/__PGSQL_VERSION__/${PGSQL_VERSION}/g" \
+    -e "s/__MSSQL_SQL__/${MSSQL_SQL}/" \
+    -e "s/__MSSQL_SETUP__/${MSSQL_SETUP}/" \
+    -e "s/__REDIS_VERSION__/${REDIS_VERSION}/" \
+    -e "s/__LDAP_SCHEMA__/${LDAP_SCHEMA}/" \
+    -e "s/__LDAP_SETUP__/${LDAP_SETUP}/" \
+    -e "s/__LDAP_VERSION__/${LDAP_VERSION}/" \
+    -e "s/__RIAK_SSL_CFG__/${RIAK_SSL_CFG}/" \
+    -e "s/__RIAK_ADV_CFG__/${RIAK_ADV_CFG}/" \
+    -e "s/__RIAK_SETUP__/${RIAK_SETUP}/" \
+    -e "s/__RIAK_SETUP_SH__/${RIAK_SETUP_SH}/" \
+    -e "s/__RIAK_MAM_SEARCH_SCHEMA__/${RIAK_MAM_SEARCH_SCHEMA}/" \
+    -e "s/__RIAK_VCARD_SEARCH_SCHEMA__/${RIAK_VCARD_SEARCH_SCHEMA}/" \
+    -e "s/__CASSA_PROXY_CNF__/${CASSA_PROXY_CNF}/" \
+    -e "s/__CASSA_ENTRY__/${CASSA_ENTRY}/" \
+    -e "s/__CASSA_MIM_SQL__/${CASSA_MIM_CQL_ENTRY}/" \
+    -e "s/__CASSA_TEST_SQL__/${CASSA_TEST_CQL_ENTRY}/" \
+    -e "s/__CASSA_VERSION__/${CASSANDRA_VERSION}/" \
+    -e "s/__ELASTICSEARCH_VERSION__/${ELASTICSEARCH_VERSION}/" \
+    -e "s/__RMQ_VERSION__/${RMQ_VERSION}/" \
+    -e "s/__MIM_CERT__/${MIM_CERT}/" \
+    -e "s/__MIM_KEY__/${MIM_KEY}/" \
+    -e "s/__MIM_PRIV_KEY__/${MIM_PRIV_KEY}/" \
+    -e "s/__MIM_DHSERVER__/${MIM_DHSERVER}/" \
+    -e "s/__INJECT_FILES__/${INJECT_FILES}/" \
+    -e "s/__DB_CACERT__/${CACERT}/" \
+    -e "s/__PYTHON2_BASE32_DEC__/${PYTHON2_BASE32_DEC}/" \
+    -e "s/__PYTHON3_BASE32_DEC__/${PYTHON3_BASE32_DEC}/" \
+    -e "s/__CERTS_CACHE_KEY__/${CERTS_CACHE_KEY}/" \
+    .circleci/template.yml \
+    > "$OUT_FILE"

tools/circle-publish-github-comment.sh

@@ -200,7 +200,7 @@ COMMENT_ID="$1"
 echo "Patch comment $COMMENT_ID"
 BODY_FROM_GH="$(cat /tmp/gh_comment | jq -r .body)"
 BODY="${BODY_FROM_GH}"$'\n'$'\n'"---"$'\n'$'\n'"${BODY}"
-PATCH_BODY=$(BODY_ENV="${BODY}" jq -n '{body: env.BODY_ENV}')
+PATCH_BODY=$(echo "${BODY}" | jq -Rn '{body: [inputs] | join("\n")}')
 curl -o /dev/null -i \
     -H "Authorization: token $COMMENTER_GITHUB_TOKEN" \
     -H "Content-Type: application/json" \
@@ -210,10 +210,24 @@ curl -o /dev/null -i \
 
 # List comments
 # https://developer.github.com/v3/issues/comments/#list-comments-on-an-issue
-curl -s -S -o /tmp/gh_comments -L \
-    -H "Authorization: token $COMMENTER_GITHUB_TOKEN" \
-    -H "Content-Type: application/json" \
-    https://api.github.com/repos/$REPO_SLUG/issues/$PR_NUM/comments
+
+function get_comments_page {
+    PAGE=$1
+    FILE=/tmp/gh_comments_page$1
+    curl -s -S -o $FILE -L \
+        -H "Authorization: token $COMMENTER_GITHUB_TOKEN" \
+        -H "Content-Type: application/json" \
+        "https://api.github.com/repos/$REPO_SLUG/issues/$PR_NUM/comments?per_page=100&page=$PAGE"
+    if test "$(cat "$FILE" | jq length)" = "100"; then
+        # Get next page
+        ((PAGE=PAGE+1))
+        get_comments_page $PAGE
+    fi
+}
+
+get_comments_page 1
+# Add all comments into one document
+jq -s "map(.) | add" /tmp/gh_comments_page* > /tmp/gh_comments
 
 # Filter out all comments for a particular user
 # Then filter out all comments that have a git commit rev in the body text

tools/circle-wait-for-db.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -e
+
+DBS=$(./tools/test_runner/presets_to_dbs.sh "$PRESET")
+echo "Wait for $DBS"
+
+function wait_for_db {
+    case $1 in
+        mysql)
+            ./tools/wait-for-it.sh -p 3306
+        ;;
+
+        pgsql)
+            ./tools/wait-for-it.sh -p 5432
+        ;;
+
+        mssql)
+            ./tools/wait-for-it.sh -p 1433
+            ./tools/wait-for-it.sh -p 1434 # SCHEMA_READY_PORT
+        ;;
+
+        rmq)
+            ./tools/wait-for-it.sh -p 5672
+        ;;
+
+        redis)
+            ./tools/wait-for-it.sh -p 6379
+        ;;
+
+        riak)
+            ./tools/wait-for-it.sh -p 8098
+            ./tools/wait-for-it.sh -p 8087
+            ./tools/wait-for-it.sh -p 8999 # SCHEMA_READY_PORT
+        ;;
+
+        ldap)
+            ./tools/wait-for-it.sh -p 636 # On Circle CI
+        ;;
+
+        elasticsearch)
+            ./tools/wait-for-it.sh -p 9200
+        ;;
+
+        cassandra)
+            ./tools/wait-for-it.sh -p 9242 # SCHEMA_READY_PORT
+            ./tools/wait-for-it.sh -p 9142 # proxy
+        ;;
+    esac
+}
+
+for db in ${DBS}; do
+    wait_for_db $db
+done

tools/circle-wait-for-solr.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+function riak_solr_is_up
+{
+    curl 'http://localhost:8093/internal_solr/mam/admin/ping?wt=json' | grep '"status":"OK"'
+}
+
+set -e
+
+# Wait for solr
+for i in {1..60}; do
+    if riak_solr_is_up; then
+        exit 0
+    fi
+    echo -n "."
+    sleep 1
+done
+
+
+echo "SOLR is not up"
+exit 1

tools/circleci-upload-to-s3.sh

@@ -24,7 +24,7 @@ if which aws ; then
     echo "aws tool ready"
 else
     curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-    unzip awscliv2.zip
+    unzip -q awscliv2.zip
     sudo ./aws/install
 fi
 

tools/db-versions.sh

@@ -7,3 +7,9 @@ ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-5.6.9}
 MYSQL_VERSION=${MYSQL_VERSION:-8.0.20}
 
 PGSQL_VERSION=${PGSQL_VERSION:-latest}
+
+LDAP_VERSION=${LDAP_VESRION:-1.2.4}
+
+REDIS_VERSION=${REDIS_VERSION:-6.2.6}
+
+RMQ_VERSION=${RMQ_VERSION:-3.7-alpine}