test: added tls to e2e-tests

- accessed clickhouse using TLS params

cmd/e2e-test/clickhouse.go

@@ -4,13 +4,12 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/msaf1980/go-stringutils"
 )
@@ -20,16 +19,19 @@ var ClickhouseOldImage = "yandex/clickhouse-server"
 var ClickhouseDefaultImage = "clickhouse/clickhouse-server"
 
 type Clickhouse struct {
-	Version string `toml:"version"`
-	Dir     string `toml:"dir"`
+	Version    string `toml:"version"`
+	Dir        string `toml:"dir"`
+	TLSEnabled bool   `toml:"tls"`
 
 	DockerImage string `toml:"image"`
 
 	TZ string `toml:"tz"` // override timezone
 
-	httpAddress string `toml:"-"`
-	url         string `toml:"-"`
-	container   string `toml:"-"`
+	httpAddress  string `toml:"-"`
+	httpsAddress string `toml:"-"`
+	url          string `toml:"-"`
+	tlsurl       string `toml:"-"`
+	container    string `toml:"-"`
 }
 
 func (c *Clickhouse) CheckConfig(rootDir string) error {
@@ -71,6 +73,7 @@ func (c *Clickhouse) Start() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	port := strings.Split(c.httpAddress, ":")[1]
 	c.url = "http://" + c.httpAddress
 
 	c.container = ClickhouseContainerName
@@ -80,14 +83,28 @@ func (c *Clickhouse) Start() (string, error) {
 	chStart := []string{"run", "-d",
 		"--name", c.container,
 		"--ulimit", "nofile=262144:262144",
-		"-p", c.httpAddress + ":8123",
+		"-p", port + ":8123",
 		// "-e", "TZ=" + tz, // workaround for TZ=":/etc/localtime"
 		"-v", c.Dir + "/config.xml:/etc/clickhouse-server/config.xml",
 		"-v", c.Dir + "/users.xml:/etc/clickhouse-server/users.xml",
 		"-v", c.Dir + "/rollup.xml:/etc/clickhouse-server/config.d/rollup.xml",
 		"-v", c.Dir + "/init.sql:/docker-entrypoint-initdb.d/init.sql",
 		"--network", DockerNetwork,
 	}
+	if c.TLSEnabled {
+		c.httpsAddress, err = getFreeTCPPort("")
+		if err != nil {
+			return "", err
+		}
+		port = strings.Split(c.httpsAddress, ":")[1]
+		c.tlsurl = "https://" + c.httpsAddress
+		chStart = append(chStart,
+			"-v", c.Dir+"/server.crt:/etc/clickhouse-server/server.crt",
+			"-v", c.Dir+"/server.key:/etc/clickhouse-server/server.key",
+			"-v", c.Dir+"/rootCA.crt:/etc/clickhouse-server/rootCA.crt",
+			"-p", port+":8443",
+		)
+	}
 	if c.TZ != "" {
 		chStart = append(chStart, "-e", "TZ="+c.TZ)
 	}
@@ -137,6 +154,10 @@ func (c *Clickhouse) URL() string {
 	return c.url
 }
 
+func (c *Clickhouse) TLSURL() string {
+	return c.tlsurl
+}
+
 func (c *Clickhouse) Container() string {
 	return c.container
 }
@@ -152,14 +173,11 @@ func (c *Clickhouse) Query(sql string) (string, error) {
 		return "", err
 	}
 
-	httpClient := http.Client{
-		Timeout: time.Minute,
-	}
-	resp, err := httpClient.Do(request)
+	resp, err := http.DefaultClient.Do(request)
 	if err != nil {
 		return "", err
 	}
-	msg, err := io.ReadAll(resp.Body)
+	msg, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return "", err
 	}
@@ -173,7 +191,7 @@ func (c *Clickhouse) Alive() bool {
 	if len(c.container) == 0 {
 		return false
 	}
-	req, err := http.DefaultClient.Get(c.url)
+	req, err := http.DefaultClient.Get(c.URL())
 	if err != nil {
 		return false
 	}

cmd/e2e-test/e2etesting.go

@@ -12,9 +12,10 @@ import (
 	"strings"
 	"time"
 
+	"go.uber.org/zap"
+
 	"github.com/lomik/graphite-clickhouse/helper/client"
 	"github.com/lomik/graphite-clickhouse/helper/datetime"
-	"go.uber.org/zap"
 
 	"github.com/pelletier/go-toml"
 )
@@ -152,6 +153,10 @@ type TestSchema struct {
 	// input map[string][]Point `toml:"-"`
 }
 
+func (schema *TestSchema) HasTLSSettings() bool {
+	return strings.Contains(schema.dir, "tls")
+}
+
 func getFreeTCPPort(name string) (string, error) {
 	if len(name) == 0 {
 		name = "127.0.0.1:0"
@@ -202,7 +207,7 @@ func sendPlain(network, address string, metrics []InputMetric) error {
 
 func verifyGraphiteClickhouse(test *TestSchema, gch *GraphiteClickhouse, clickhouse *Clickhouse, testDir, clickhouseDir string, verbose, breakOnError bool, logger *zap.Logger) (testSuccess bool, verifyCount, verifyFailed int) {
 	testSuccess = true
-	err := gch.Start(testDir, clickhouse.URL(), test.Proxy.URL())
+	err := gch.Start(testDir, clickhouse.URL(), clickhouse.TLSURL(), test.Proxy.URL())
 	if err != nil {
 		logger.Error("starting graphite-clickhouse",
 			zap.String("config", test.name),

cmd/e2e-test/graphite-clickhouse.go

@@ -12,13 +12,15 @@ import (
 	"syscall"
 	"text/template"
 
-	"github.com/lomik/graphite-clickhouse/helper/client"
 	"github.com/msaf1980/go-stringutils"
+
+	"github.com/lomik/graphite-clickhouse/helper/client"
 )
 
 type GraphiteClickhouse struct {
 	Binary    string `toml:"binary"`
 	ConfigTpl string `toml:"template"`
+	TestDir   string `toml:"-"`
 
 	TZ string `toml:"tz"` // override timezone
 
@@ -28,7 +30,7 @@ type GraphiteClickhouse struct {
 	cmd        *exec.Cmd `toml:"-"`
 }
 
-func (c *GraphiteClickhouse) Start(testDir, clickhouseURL, chProxyURL string) error {
+func (c *GraphiteClickhouse) Start(testDir, chURL, chProxyURL, chTLSURL string) error {
 	if c.cmd != nil {
 		return errors.New("carbon-clickhouse already started")
 	}
@@ -52,22 +54,31 @@ func (c *GraphiteClickhouse) Start(testDir, clickhouseURL, chProxyURL string) er
 		return err
 	}
 
+	c.TestDir, err = filepath.Abs(testDir)
+	if err != nil {
+		return err
+	}
+
 	name := filepath.Base(c.ConfigTpl)
 	tmpl, err := template.New(name).ParseFiles(path.Join(testDir, c.ConfigTpl))
 	if err != nil {
 		c.Cleanup()
 		return err
 	}
 	param := struct {
-		CLICKHOUSE_URL string
-		PROXY_URL      string
-		GCH_ADDR       string
-		GCH_DIR        string
+		CLICKHOUSE_URL     string
+		CLICKHOUSE_TLS_URL string
+		PROXY_URL          string
+		GCH_ADDR           string
+		GCH_DIR            string
+		TEST_DIR           string
 	}{
-		CLICKHOUSE_URL: clickhouseURL,
-		PROXY_URL:      chProxyURL,
-		GCH_ADDR:       c.address,
-		GCH_DIR:        c.storeDir,
+		CLICKHOUSE_URL:     chURL,
+		CLICKHOUSE_TLS_URL: chTLSURL,
+		PROXY_URL:          chProxyURL,
+		GCH_ADDR:           c.address,
+		GCH_DIR:            c.storeDir,
+		TEST_DIR:           c.TestDir,
 	}
 
 	c.configFile = path.Join(c.storeDir, "graphite-clickhouse.conf")

config/config.go

@@ -139,8 +139,6 @@ func binarySearchQueryParamLe(a []QueryParam, duration time.Duration, start, end
 // ClickHouse config
 type ClickHouse struct {
 	URL                   string                `toml:"url"                      json:"url"                      comment:"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params"`
-	TLSParams             config.TLS            `toml:"tls"                      json:"tls"                      comment:"mTLS HTTPS configuration for connecting to clickhouse server"                                                                         commented:"true"`
-	TLSConfig             *tls.Config           `toml:"-"                        json:"-"`
 	DataTimeout           time.Duration         `toml:"data-timeout"             json:"data-timeout"             comment:"default total timeout to fetch data, can be overwritten with query-params"`
 	RenderMaxQueries      int                   `toml:"render-max-queries"       json:"render-max-queries"       comment:"Max queries to render queiries"`
 	RenderMaxConcurrent   int                   `toml:"render-max-concurrent"    json:"render-max-concurrent"    comment:"Maximum concurrent queries to render queiries"`
@@ -179,6 +177,9 @@ type ClickHouse struct {
 	MaxDataPoints    int    `toml:"max-data-points"          json:"max-data-points"          comment:"max points per metric when internal-aggregation=true"`
 	// InternalAggregation controls if ClickHouse itself or graphite-clickhouse aggregates points to proper retention
 	InternalAggregation bool `toml:"internal-aggregation"     json:"internal-aggregation"     comment:"ClickHouse-side aggregation, see doc/aggregation.md"`
+
+	TLSParams config.TLS  `toml:"tls"                      json:"tls"                      comment:"mTLS HTTPS configuration for connecting to clickhouse server"                                                                         commented:"true"`
+	TLSConfig *tls.Config `toml:"-"                        json:"-"`
 }
 
 func clickhouseURLValidate(chURL string) (*url.URL, error) {

doc/config.md

@@ -237,17 +237,6 @@ Only one tag used as filter for index field Tag1, see graphite_tagged table [str
 [clickhouse]
  # default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params
  url = "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1"
-
- # mTLS HTTPS configuration for connecting to clickhouse server
- # [clickhouse.tls]
-  # ca-cert = []
-  # client-auth = ""
-  # server-name = ""
-  # min-version = ""
-  # max-version = ""
-  # insecure-skip-verify = false
-  # curves = []
-  # cipher-suites = []
  # default total timeout to fetch data, can be overwritten with query-params
  data-timeout = "1m0s"
  # Max queries to render queiries
@@ -325,6 +314,17 @@ Only one tag used as filter for index field Tag1, see graphite_tagged table [str
  # ClickHouse-side aggregation, see doc/aggregation.md
  internal-aggregation = true
 
+ # mTLS HTTPS configuration for connecting to clickhouse server
+ # [clickhouse.tls]
+  # ca-cert = []
+  # client-auth = ""
+  # server-name = ""
+  # min-version = ""
+  # max-version = ""
+  # insecure-skip-verify = false
+  # curves = []
+  # cipher-suites = []
+
 [[data-table]]
  # data table from carbon-clickhouse
  table = "graphite_data"

tests/clickhouse/rollup_tls/config.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<yandex>
+  <logger>
+    <level>debug</level>
+    <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+    <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+    <size>2000M</size>
+    <count>20</count>
+  </logger>
+  <!-- http  port and tcp port are left open because they are required for checking if 
+  clickhouse is still running -->
+  <http_port>8123</http_port>
+  <tcp_port>9000</tcp_port>
+  <https_port>8443</https_port>
+  <tcp_port_secure>9440</tcp_port_secure>
+  <openSSL>
+    <server>
+      <verificationMode>none</verificationMode>
+      <loadDefaultCAFile>false</loadDefaultCAFile>
+      <caConfig>/etc/clickhouse-server/rootCA.crt</caConfig>
+      <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
+      <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
+      <cacheSessions>true</cacheSessions>
+      <!-- <disableProtocols>sslv2,sslv3</disableProtocols> -->
+      <preferServerCiphers>true</preferServerCiphers>
+    </server>
+    <client>
+      <caConfig>/etc/clickhouse-server/rootCA.crt</caConfig>
+    </client>
+  </openSSL>
+
+  <!-- Port for communication between replicas. Used for data exchange. -->
+  <interserver_http_port>9009</interserver_http_port>
+
+  <interserver_http_host>test-clickhouse-s1</interserver_http_host>
+
+
+  <!-- Listen specified host. use :: (wildcard IPv6 address), if you want to accept connections both
+  with IPv4 and IPv6 from everywhere. -->
+  <!-- <listen_host>::</listen_host> -->
+  <!-- <listen_host>::1</listen_host> -->
+  <listen_host>0.0.0.0</listen_host>
+
+
+  <uncompressed_cache_size>1073741824</uncompressed_cache_size>
+
+  <!-- Approximate size of mark cache, used in tables of MergeTree family.
+         In bytes. Cache is single for server. Memory is allocated only on demand.
+         You should not lower this value.
+      -->
+  <mark_cache_size>1073741824</mark_cache_size>
+
+  <!-- Path to data directory, with trailing slash. -->
+  <path>/var/lib/clickhouse/</path>
+
+  <!-- Path to temporary data for processing hard queries. -->
+  <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+
+  <!-- Path to configuration file with users, access rights, profiles of settings, quotas. -->
+  <users_config>users.xml</users_config>
+
+  <!-- Default profile of settings.. -->
+  <default_profile>default</default_profile>
+
+  <!-- Default database. -->
+  <default_database>default</default_database>
+
+  <!-- Query log. Used only for queries with setting log_queries = 1. -->
+  <query_log>
+    <!-- What table to insert data. If table is not exist, it will be created.
+             When query log structure is changed after system update,
+              then old table will be renamed and new table will be created automatically.
+        -->
+    <database>system</database>
+    <table>query_log</table>
+
+    <!-- Interval of flushing data. -->
+    <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+  </query_log>
+
+
+  <!-- Uncomment if use part_log -->
+  <part_log>
+    <database>system</database>
+    <table>part_log</table>
+
+    <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+  </part_log>
+  <!-- -->
+
+</yandex>

tests/clickhouse/rollup_tls/init.sql

@@ -0,0 +1,38 @@
+CREATE TABLE IF NOT EXISTS default.graphite_reverse (
+  Path String,
+  Value Float64,
+  Time UInt32,
+  Date Date,
+  Timestamp UInt32
+) ENGINE = GraphiteMergeTree('graphite_rollup')
+PARTITION BY toYYYYMM(Date)
+ORDER BY (Path, Time);
+
+CREATE TABLE IF NOT EXISTS default.graphite (
+  Path String,
+  Value Float64,
+  Time UInt32,
+  Date Date,
+  Timestamp UInt32
+) ENGINE = GraphiteMergeTree('graphite_rollup')
+PARTITION BY toYYYYMM(Date)
+ORDER BY (Path, Time);
+
+CREATE TABLE IF NOT EXISTS default.graphite_index (
+  Date Date,
+  Level UInt32,
+  Path String,
+  Version UInt32
+) ENGINE = ReplacingMergeTree(Version)
+PARTITION BY toYYYYMM(Date)
+ORDER BY (Level, Path, Date);
+
+CREATE TABLE IF NOT EXISTS default.graphite_tags (
+  Date Date,
+  Tag1 String,
+  Path String,
+  Tags Array(String),
+  Version UInt32
+) ENGINE = ReplacingMergeTree(Version)
+PARTITION BY toYYYYMM(Date)
+ORDER BY (Tag1, Path, Date);