Merge 59da76f into e108c81

.travis.yml

@@ -53,6 +53,8 @@ after_script:
             tools/travis-prepare-log-dir.sh
         - test 1 = "$SKIP_REPORT_UPLOAD" ||
             if [ -n "${AWS_SECRET_ACCESS_KEY}" ]; then tools/travis-upload-to-s3.sh; fi
+        - test 1 = "$SKIP_REPORT_UPLOAD" ||
+            tools/travis-publish-github-comment.sh
         - test 1 = "$SKIP_REPORT_UPLOAD" ||
             if [ -n "${GDRIVE_SERVICE_ACCOUNT_CREDENTIALS}" ]; then tools/travis-upload-to-gdrive.sh; fi
         - test 1 = "$SKIP_REPORT_UPLOAD" ||

big_tests/default.spec

@@ -82,6 +82,7 @@
 %% * ensure preset value is passed to ct Config
 %% * check server's purity after SUITE
 {ct_hooks, [ct_tty_hook, ct_mongoose_hook, ct_progress_hook,
+            ct_markdown_errors_hook,
             {ct_mongoose_log_hook, [ejabberd_node, ejabberd_cookie]},
             {ct_mongoose_log_hook, [ejabberd2_node, ejabberd_cookie]}
            ]}.

big_tests/src/ct_markdown_errors_hook.erl

@@ -0,0 +1,151 @@
+%%% @doc Writes a markdown file with error information
+-module(ct_markdown_errors_hook).
+
+%% @doc Add the following line in your *.spec file to enable this:
+%% {ct_hooks, [ct_markdown_errors_hook]}.
+
+%% Callbacks
+-export([id/1]).
+-export([init/2]).
+-export([post_init_per_suite/4,
+         post_init_per_group/4,
+         post_init_per_testcase/4]).
+-export([post_end_per_suite/4,
+         post_end_per_group/4,
+         post_end_per_testcase/4]).
+-record(state, { file, suite, group, limit }).
+
+%% @doc Return a unique id for this CTH.
+id(_Opts) ->
+    "ct_markdown_errors_hook_001".
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state.
+init(_Id, _Opts) ->
+    File = "/tmp/ct_markdown",
+    file:write_file(File, "", []),
+    {ok, #state{ file = File, limit = 25 }}.
+
+post_init_per_suite(SuiteName, Config, Return, State) ->
+    State2 = handle_return(SuiteName, '', init_per_suite, Return, Config, State),
+    {Return, State2#state{group = '', suite = SuiteName}}.
+
+post_init_per_group(GroupName, Config, Return, State=#state{suite = SuiteName}) ->
+    State2 = handle_return(SuiteName, GroupName, init_per_group, Return, Config, State),
+    {Return, State2#state{group = GroupName}}.
+
+post_init_per_testcase(TC, Config, Return, State=#state{group = GroupName,
+                                                         suite = SuiteName}) ->
+    State2 = handle_return(SuiteName, GroupName, TC, Return, Config, State),
+    {Return, State2}.
+
+post_end_per_suite(SuiteName, Config, Return, State) ->
+    State2 = handle_return(SuiteName, '', end_per_suite, Return, Config, State),
+    {Return, State2#state{suite = '', group = ''}}.
+
+post_end_per_group(GroupName, Config, Return, State=#state{suite = SuiteName}) ->
+    State2 = handle_return(SuiteName, GroupName, end_per_group, Return, Config, State),
+    {Return, State2#state{group = ''}}.
+
+%% @doc Called after each test case.
+post_end_per_testcase(TC, Config, Return, State=#state{group = GroupName,
+                                                        suite = SuiteName}) ->
+    State2 = handle_return(SuiteName, GroupName, TC, Return, Config, State),
+    {Return, State2}.
+
+handle_return(SuiteName, GroupName, Place, Return, Config, State) ->
+    try handle_return_unsafe(SuiteName, GroupName, Place, Return, Config, State)
+    catch Class:Error ->
+        Stacktrace = erlang:get_stacktrace(),
+        ct:pal("issue=handle_return_unsafe_failed reason=~p:~p~n"
+               "stacktrace=~p", [Class, Error, Stacktrace]),
+        State
+    end.
+
+handle_return_unsafe(SuiteName, GroupName, Place, Return, Config, State) ->
+    case to_error_message(Return) of
+        ok ->
+            State;
+        Error ->
+            F = fun() ->
+                log_error(SuiteName, GroupName, Place, Error, Config, State)
+                end,
+            exec_limited_number_of_times(F, State)
+    end.
+
+exec_limited_number_of_times(F, State=#state{limit=0, file=File}) ->
+    %% Log truncated
+    file:write_file(File, ".", [append]),
+    State;
+exec_limited_number_of_times(F, State=#state{limit=Limit}) ->
+    F(),
+    State#state{limit=Limit-1}.
+
+log_error(SuiteName, GroupName, Place, Error, Config, #state{file = File}) ->
+    MaybeLogLink = make_log_link(Config),
+    LogLink = make_log_link(Config),
+    %% Spoler syntax
+    %% https://github.com/dear-github/dear-github/issues/166
+    %%    <details>
+    %%      <summary>Click to expand</summary>
+    %%      whatever
+    %%    </details>
+    SummaryText = make_summary_text(SuiteName, GroupName, Place),
+    BinError = iolist_to_binary(io_lib:format("~p", [Error])),
+    Content = truncate_binary(1500, reindent(BinError)),
+    %% Don't use exml here to avoid escape errors
+    Out = <<"<details><summary>", SummaryText/binary, "</summary>\n"
+            "\n\n```erlang\n", Content/binary, "\n```\n",
+            LogLink/binary, "</details>\n">>,
+    file:write_file(File, Out, [append]),
+    ok.
+
+make_summary_text(SuiteName, '', '') ->
+     atom_to_binary(SuiteName, utf8);
+make_summary_text(SuiteName, '', TC) ->
+    BSuiteName = atom_to_binary(SuiteName, utf8),
+    BTC = atom_to_binary(TC, utf8),
+    <<BSuiteName/binary, ":", BTC/binary>>;
+make_summary_text(SuiteName, GroupName, TC) ->
+    BSuiteName = atom_to_binary(SuiteName, utf8),
+    BGroupName = atom_to_binary(GroupName, utf8),
+    BTC = atom_to_binary(TC, utf8),
+    <<BSuiteName/binary, ":", BGroupName/binary, ":", BTC/binary>>.
+
+make_log_link(Config) ->
+    LogFile = proplists:get_value(tc_logfile, Config, ""),
+    LogLink =
+    case LogFile of
+        "" ->
+            <<>>;
+        _ ->
+            <<"\n[Report log](", (list_to_binary(LogFile))/binary, ")\n">>
+    end.
+
+to_error_message(Return) ->
+    case Return of
+        {'EXIT', _} ->
+            Return;
+        {fail, _} ->
+            Return;
+        {error, _} ->
+            Return;
+        {skip, _} ->
+            ok;
+        _ ->
+            ok
+    end.
+
+truncate_binary(Len, Bin) ->
+    case byte_size(Bin) > Len of
+        true ->
+            Prefix = binary:part(Bin, {0,Len}),
+            <<Prefix/binary, "...">>;
+        false ->
+            Bin
+    end.
+
+reindent(Bin) ->
+    %% Use 2 whitespaces instead of 4 for indention
+    %% to make more compact look
+    binary:replace(Bin, <<"    ">>, <<"  ">>, [global]).

tools/travis-publish-github-comment.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+
+source tools/travis-helpers.sh
+
+set -e
+
+if [ -z "$TRAVIS_PULL_REQUEST" ]; then
+    echo "\$TRAVIS_PULL_REQUEST is empty. Do nothing"
+    exit 0
+fi
+
+if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
+    echo "Not a pull request. Do nothing"
+    exit 0
+fi
+
+if [ -z "$COMMENTER_GITHUB_TOKEN" ]; then
+    echo "\$COMMENTER_GITHUB_TOKEN is empty. Do nothing"
+    exit 0
+fi
+
+# COMMENTER_GITHUB_USER is nickname of a special github user.
+# COMMENTER_GITHUB_TOKEN is token with scope public repo.
+# Token can be obtained here https://github.com/settings/tokens/new
+# Don't use ANY REAL user as a commenter, because GitHub access scopes are too wide.
+echo "Used vars (you can use them to debug locally):"
+echo "export COMMENTER_GITHUB_TOKEN=fillme"
+echo "export COMMENTER_GITHUB_USER=$COMMENTER_GITHUB_USER"
+echo "export PRESET=$PRESET"
+echo "export TRAVIS_COMMIT=$TRAVIS_COMMIT"
+echo "export TRAVIS_JOB_ID=$TRAVIS_JOB_ID"
+echo "export TRAVIS_JOB_NUMBER=$TRAVIS_JOB_NUMBER"
+echo "export TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG"
+echo "export TRAVIS_OTP_RELEASE=$TRAVIS_OTP_RELEASE"
+echo "export TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST"
+
+PRESET="${PRESET:-default}"
+TRAVIS_OTP_RELEASE="${TRAVIS_OTP_RELEASE:-unknown}"
+
+function remove_ct_log_links
+{
+    mv /tmp/ct_markdown /tmp/ct_markdown_original
+    grep -v "Report log" /tmp/ct_markdown_original > /tmp/ct_markdown
+}
+
+# https://stackoverflow.com/questions/407523/escape-a-string-for-a-sed-replace-pattern
+function replace_string {
+    sed -i "s/$( \
+        echo "$1" | sed -e 's/\([[\/.*]\|\]\)/\\&/g' \
+                  | sed -e 's:\t:\\t:g' \
+    )/$( \
+        echo "$2" | sed -e 's/[\/&]/\\&/g' \
+                  | sed -e 's:\t:\\t:g' \
+    )/g" "$3"
+}
+
+function rewrite_log_links_to_s3
+{
+    # REPORTS_URL is
+    # http://esl.github.io/mongooseim-ct-reports/s3_reports.html?prefix=PR/1916/4855/ldap_mnesia.19.3
+    # redirects to
+    # http://mongooseim-ct-results.s3-eu-west-1.amazonaws.com/PR/1885/4744/mysql_redis.19.3/big/
+    REPORTS_URL="$1"
+    CT_REPORT=big_tests/ct_report
+    CT_REPORT_ABS=$(./tools/abs_dirpath.sh "$CT_REPORT")
+    S3_REPORT="$REPORTS_URL/big"
+    cp /tmp/ct_markdown /tmp/ct_markdown_original
+    replace_string "$CT_REPORT_ABS" "$S3_REPORT" /tmp/ct_markdown
+    # URL escape for s3_reports.html script
+    replace_string "ct_run.test@" "ct_run.test%40" /tmp/ct_markdown
+}
+
+if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
+    REPORTS_URL_BODY="Reports are not uploaded"$'\n'
+    remove_ct_log_links
+    rewrite_log_links_to_s3 "test"
+else
+    CT_REPORTS=$(ct_reports_dir)
+    REPORTS_URL=$(s3_url ${CT_REPORTS})
+    REPORTS_URL_BODY="[Reports URL](${REPORTS_URL})"$'\n'
+    rewrite_log_links_to_s3 "$REPORTS_URL"
+fi
+
+# Link to a travis job
+JOB_URL="https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID"
+DESC_BODY="[$TRAVIS_JOB_NUMBER]($JOB_URL) / Erlang $TRAVIS_OTP_RELEASE / $PRESET / $TRAVIS_COMMIT"$'\n'
+# This file is created by ct_markdown_errors_hook
+ERRORS_BODY="$(cat /tmp/ct_markdown || echo '/tmp/ct_markdown missing')"
+BODY="${DESC_BODY}${REPORTS_URL_BODY}${ERRORS_BODY}"
+# SLUG is the same for both GitHub and Travis CI
+TRAVIS_REPO_SLUG=${TRAVIS_REPO_SLUG:-esl/MongooseIM}
+
+function post_new_comment
+{
+# Create a comment GitHub API doc
+# https://developer.github.com/v3/issues/comments/#create-a-comment
+echo "Posting a new comment"
+POST_BODY=$(BODY_ENV="$BODY" jq -n '{body: env.BODY_ENV}')
+curl -o /dev/null -i \
+    -H "Authorization: token $COMMENTER_GITHUB_TOKEN" \
+    -H "Content-Type: application/json" \
+    -X POST -d "$POST_BODY" \
+    https://api.github.com/repos/$TRAVIS_REPO_SLUG/issues/$TRAVIS_PULL_REQUEST/comments
+}
+
+function append_comment
+{
+# Concat old comment text and some extra text
+# Keep a line separator between them
+# $'\n' is a new line in bash
+#
+# Edit commment GitHub API doc
+# https://developer.github.com/v3/issues/comments/#edit-a-comment
+COMMENT_ID="$1"
+echo "Patch comment $COMMENT_ID"
+BODY_FROM_GH="$(cat /tmp/gh_comment | jq -r .body)"
+BODY="${BODY_FROM_GH}"$'\n'$'\n'"---"$'\n'$'\n'"${BODY}"
+PATCH_BODY=$(BODY_ENV="${BODY}" jq -n '{body: env.BODY_ENV}')
+curl -o /dev/null -i \
+    -H "Authorization: token $COMMENTER_GITHUB_TOKEN" \
+    -H "Content-Type: application/json" \
+    -X PATCH -d "$PATCH_BODY" \
+    https://api.github.com/repos/$TRAVIS_REPO_SLUG/issues/comments/$COMMENT_ID
+}
+
+# List comments
+# https://developer.github.com/v3/issues/comments/#list-comments-on-an-issue
+curl -s -S -o /tmp/gh_comments -L \
+    -H "Authorization: token $COMMENTER_GITHUB_TOKEN" \
+    -H "Content-Type: application/json" \
+    https://api.github.com/repos/$TRAVIS_REPO_SLUG/issues/$TRAVIS_PULL_REQUEST/comments
+
+# Filter out all comments for a particular user
+# Then filter out all comments that have a git commit rev in the body text
+# Then take first comment (we don't expect more comments anyway)
+cat /tmp/gh_comments | jq "map(select(.user.login == \"$COMMENTER_GITHUB_USER\")) | map(select(.body | contains(\"$TRAVIS_COMMIT\")))[0]" > /tmp/gh_comment
+COMMENT_ID=$(cat /tmp/gh_comment | jq .id)
+
+if [ "$COMMENT_ID" = "null" ]; then
+    post_new_comment
+else
+    append_comment "$COMMENT_ID"
+fi