Make SQL queries clickable in Live Tail Drilldown (#511)

After this change each SQL query in the "Translated SQL" view on Live
Tail Drilldown page is now clickable and routes to the ClickHouse
`/play` webpage with the query pre-filled-in, which makes it very
convenient to quickly run and modify it.

This change required changing the way query bodies are stored internally
- previously the queries were concatenated to a single large blob, but
now each query is stored separately (as a `[]byte`), making it possible
to display them separately in the UI.

quesma/quesma/functionality/terms_enum/terms_enum.go

@@ -102,7 +102,7 @@ func handleTermsEnumRequest(ctx context.Context, body types.JSON, qt *queryparse
 		Id:                     ctx.Value(tracing.RequestIdCtxKey).(string),
 		Path:                   path,
 		IncomingQueryBody:      reqBody,
-		QueryBodyTranslated:    []byte(selectQuery.SelectCommand.String()),
+		QueryBodyTranslated:    [][]byte{[]byte(selectQuery.SelectCommand.String())},
 		QueryTranslatedResults: result,
 		SecondaryTook:          time.Since(startTime),
 	})

quesma/quesma/search.go

@@ -134,7 +134,7 @@ func (q *QueryRunner) handleAsyncSearch(ctx context.Context, indexPattern string
 
 type AsyncSearchWithError struct {
 	response            *model.SearchResp
-	translatedQueryBody []byte
+	translatedQueryBody [][]byte
 	err                 error
 }
 
@@ -277,14 +277,16 @@ func (q *QueryRunner) handleSearchCommon(ctx context.Context, indexPattern strin
 			}()
 
 		} else {
-			queriesBody := ""
-			for _, query := range queries {
-				queriesBody += query.SelectCommand.String() + "\n"
+			queriesBody := make([][]byte, len(queries))
+			queriesBodyConcat := ""
+			for i, query := range queries {
+				queriesBody[i] = []byte(query.SelectCommand.String())
+				queriesBodyConcat += query.SelectCommand.String() + "\n"
 			}
 			responseBody = []byte(fmt.Sprintf("Invalid Queries: %s, err: %v", queriesBody, err))
-			logger.ErrorWithCtxAndReason(ctx, "Quesma generated invalid SQL query").Msg(queriesBody)
+			logger.ErrorWithCtxAndReason(ctx, "Quesma generated invalid SQL query").Msg(queriesBodyConcat)
 			bodyAsBytes, _ := body.Bytes()
-			pushSecondaryInfo(q.quesmaManagementConsole, id, path, bodyAsBytes, []byte(queriesBody), responseBody, startTime)
+			pushSecondaryInfo(q.quesmaManagementConsole, id, path, bodyAsBytes, queriesBody, responseBody, startTime)
 			return responseBody, errors.New(string(responseBody))
 		}
 
@@ -556,9 +558,9 @@ func (q *QueryRunner) runQueryJobs(jobs []QueryJob) ([][]model.QueryResultRow, e
 func (q *QueryRunner) searchWorkerCommon(
 	ctx context.Context,
 	queries []*model.Query,
-	table *clickhouse.Table) (translatedQueryBody []byte, hits [][]model.QueryResultRow, err error) {
-	sqls := ""
+	table *clickhouse.Table) (translatedQueryBody [][]byte, hits [][]model.QueryResultRow, err error) {
 
+	translatedQueryBody = make([][]byte, len(queries))
 	hits = make([][]model.QueryResultRow, len(queries))
 
 	var jobs []QueryJob
@@ -573,7 +575,7 @@ func (q *QueryRunner) searchWorkerCommon(
 
 		sql := query.SelectCommand.String()
 		logger.InfoWithCtx(ctx).Msgf("SQL: %s", sql)
-		sqls += sql + "\n"
+		translatedQueryBody[i] = []byte(sql)
 
 		if q.isInternalKibanaQuery(query) {
 			hits[i] = make([]model.QueryResultRow, 0)
@@ -609,15 +611,14 @@ func (q *QueryRunner) searchWorkerCommon(
 		hits[hitsPosition] = dbHits[jobId]
 	}
 
-	translatedQueryBody = []byte(sqls)
 	return
 }
 
 func (q *QueryRunner) searchWorker(ctx context.Context,
 	aggregations []*model.Query,
 	table *clickhouse.Table,
 	doneCh chan<- AsyncSearchWithError,
-	optAsync *AsyncQuery) (translatedQueryBody []byte, resultRows [][]model.QueryResultRow, err error) {
+	optAsync *AsyncQuery) (translatedQueryBody [][]byte, resultRows [][]model.QueryResultRow, err error) {
 	if optAsync != nil {
 		if q.reachedQueriesLimit(ctx, optAsync.asyncRequestIdStr, doneCh) {
 			return
@@ -674,7 +675,7 @@ func (q *QueryRunner) postProcessResults(table *clickhouse.Table, results [][]mo
 	return geoIpTransformer.Transform(res)
 }
 
-func pushSecondaryInfo(qmc *ui.QuesmaManagementConsole, Id, Path string, IncomingQueryBody, QueryBodyTranslated, QueryTranslatedResults []byte, startTime time.Time) {
+func pushSecondaryInfo(qmc *ui.QuesmaManagementConsole, Id, Path string, IncomingQueryBody []byte, QueryBodyTranslated [][]byte, QueryTranslatedResults []byte, startTime time.Time) {
 	qmc.PushSecondaryInfo(&ui.QueryDebugSecondarySource{
 		Id:                     Id,
 		Path:                   Path,

quesma/quesma/ui/asset/head.html

@@ -192,7 +192,8 @@
             background-color: rgb(221, 226, 235);
         }
 
-        .debug-body a {
+        .debug-body a,
+        .query-body-translated a {
             text-decoration: none;
         }
 
@@ -203,7 +204,8 @@
         }
 
         .right .debug-body a,
-        .bottom_left .debug-body a {
+        .bottom_left .debug-body a,
+        #request-info .query-body-translated a {
             color: white;
             display: block;
         }
@@ -214,7 +216,8 @@
         }
 
         .right .debug-body a:hover,
-        .bottom_left .debug-body a:hover {
+        .bottom_left .debug-body a:hover,
+        #request-info .query-body-translated a:hover {
             background-color: rgb(40, 40, 40);
         }
 

quesma/quesma/ui/html_pages_test.go

@@ -26,7 +26,7 @@ func TestHtmlPages(t *testing.T) {
 	qmc.PushSecondaryInfo(&QueryDebugSecondarySource{Id: id,
 		Path:                   xss,
 		IncomingQueryBody:      xssBytes,
-		QueryBodyTranslated:    xssBytes,
+		QueryBodyTranslated:    [][]byte{xssBytes},
 		QueryTranslatedResults: xssBytes,
 	})
 	log := fmt.Sprintf(`{"request_id": "%s", "message": "%s"}`, id, xss)

quesma/quesma/ui/live_tail.go

@@ -3,6 +3,7 @@
 package ui
 
 import (
+	"bytes"
 	"fmt"
 	"quesma/buildinfo"
 	"quesma/quesma/config"
@@ -246,7 +247,7 @@ func (qmc *QuesmaManagementConsole) populateQueries(debugKeyValueSlice []queryDe
 		tookStr := fmt.Sprintf(" took %d ms", v.query.SecondaryTook.Milliseconds())
 		buffer.Html("<p>UUID:").Text(v.id).Text(tookStr).Html(errorBanner(v.query)).Html("</p>\n")
 		buffer.Html(`<pre Id="second_query`).Text(v.id).Html(`">`)
-		buffer.Text(util.SqlPrettyPrint(v.query.QueryBodyTranslated))
+		buffer.Text(util.SqlPrettyPrint(bytes.Join(v.query.QueryBodyTranslated, []byte{})))
 		buffer.Html("\n</pre>")
 		if withLinks {
 			buffer.Html("\n</a>")

quesma/quesma/ui/live_tail_drilldown.go

@@ -3,6 +3,7 @@
 package ui
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"gopkg.in/yaml.v3"
@@ -44,9 +45,21 @@ func (qmc *QuesmaManagementConsole) generateReportForRequestId(requestId string)
 
 		buffer.Html(`<div class="query-body-translated">` + "\n")
 		buffer.Html("<p class=\"title\">Translated SQL:</p>\n")
-		buffer.Html(`<pre>`)
-		buffer.Text(util.SqlPrettyPrint(request.QueryBodyTranslated))
-		buffer.Html("\n</pre>")
+		for _, queryBody := range request.QueryBodyTranslated {
+			prettyQueryBody := util.SqlPrettyPrint(queryBody)
+			if qmc.cfg.ClickHouse.AdminUrl != nil {
+				// ClickHouse web UI /play expects a base64-encoded query
+				// in the URL:
+				base64QueryBody := base64.StdEncoding.EncodeToString([]byte(prettyQueryBody))
+				buffer.Html(`<a href="`).Text(qmc.cfg.ClickHouse.AdminUrl.String()).Text("/play#").Text(base64QueryBody).Html(`">`)
+			}
+			buffer.Html(`<pre>`)
+			buffer.Text(prettyQueryBody)
+			buffer.Html("\n</pre>")
+			if qmc.cfg.ClickHouse.AdminUrl != nil {
+				buffer.Html(`</a>`)
+			}
+		}
 		buffer.Html(`</div>` + "\n")
 
 		buffer.Html(`<div class="elastic-response">` + "\n")

quesma/quesma/ui/management_console.go

@@ -3,6 +3,7 @@
 package ui
 
 import (
+	"bytes"
 	"github.com/rs/zerolog"
 	"quesma/elasticsearch"
 	"quesma/schema"
@@ -47,7 +48,7 @@ type QueryDebugSecondarySource struct {
 	Path              string
 	IncomingQueryBody []byte
 
-	QueryBodyTranslated    []byte
+	QueryBodyTranslated    [][]byte
 	QueryTranslatedResults []byte
 	SecondaryTook          time.Duration
 }
@@ -134,7 +135,7 @@ func (qmc *QuesmaManagementConsole) RecordRequest(typeName string, took time.Dur
 
 func (qdi *queryDebugInfo) requestContains(queryStr string) bool {
 	potentialPlaces := [][]byte{qdi.QueryDebugSecondarySource.IncomingQueryBody,
-		qdi.QueryDebugSecondarySource.QueryBodyTranslated}
+		bytes.Join(qdi.QueryDebugSecondarySource.QueryBodyTranslated, []byte{})}
 	for _, potentialPlace := range potentialPlaces {
 		if potentialPlace != nil && strings.Contains(string(potentialPlace), queryStr) {
 			return true