Merge pull request #186 from lexx-bright/RollupAutoInterval

Make rollup update interval configurable when rollup-conf set to "auto".

config/config.go

@@ -149,6 +149,7 @@ type DataTable struct {
 	TargetMatchAllRegexp   *regexp.Regexp  `toml:"-" json:"-"`
 	RollupConf             string          `toml:"rollup-conf" json:"-" comment:"custom rollup.xml file for table, 'auto' and 'none' are allowed as well"`
 	RollupAutoTable        string          `toml:"rollup-auto-table" json:"rollup-auto-table" comment:"custom table for 'rollup-conf=auto', useful for Distributed or MatView"`
+	RollupAutoInterval     *time.Duration  `toml:"rollup-auto-interval" json:"rollup-auto-interval" comment:"rollup update interval for 'rollup-conf=auto'"`
 	RollupDefaultPrecision uint32          `toml:"rollup-default-precision" json:"rollup-default-precision" comment:"is used when none of rules match"`
 	RollupDefaultFunction  string          `toml:"rollup-default-function" json:"rollup-default-function" comment:"is used when none of rules match"`
 	RollupUseReverted      bool            `toml:"rollup-use-reverted" json:"rollup-use-reverted" comment:"should be set to true if you don't have reverted regexps in rollup-conf for reversed tables"`
@@ -272,10 +273,12 @@ func PrintDefaultConfig() error {
 	}
 
 	if len(cfg.DataTable) == 0 {
+                interval := time.Minute
 		cfg.DataTable = []DataTable{
 			{
 				Table:      "graphite_data",
 				RollupConf: "auto",
+				RollupAutoInterval: &interval,
 			},
 		}
 	}
@@ -454,11 +457,15 @@ func (c *Config) ProcessDataTables() (err error) {
 		rdf := c.DataTable[i].RollupDefaultFunction
 		if c.DataTable[i].RollupConf == "auto" || c.DataTable[i].RollupConf == "" {
 			table := c.DataTable[i].Table
+			interval := time.Minute
 			if c.DataTable[i].RollupAutoTable != "" {
 				table = c.DataTable[i].RollupAutoTable
 			}
+			if c.DataTable[i].RollupAutoInterval != nil {
+				interval = *c.DataTable[i].RollupAutoInterval
+			}
 
-			c.DataTable[i].Rollup, err = rollup.NewAuto(c.ClickHouse.URL, table, time.Minute, rdp, rdf)
+			c.DataTable[i].Rollup, err = rollup.NewAuto(c.ClickHouse.URL, table, interval, rdp, rdf)
 		} else if c.DataTable[i].RollupConf == "none" {
 			c.DataTable[i].Rollup, err = rollup.NewDefault(rdp, rdf)
 		} else {

deploy/doc/config.md

@@ -59,7 +59,7 @@ The rollup configuration is used for a proper  metrics pre-aggregation. It conta
 
 Historically, the way to define the config was `rollup-conf = "/path/to/the/conf/with/graphite_rollup.xml"`. The format is the same as [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration) scheme for ClickHouse server.
 
-For a quite long time it's recommended to use `rollup-conf = "auto"` to get the configuration from remote ClickHouse server. It will update itself each minute.
+For a quite long time it's recommended to use `rollup-conf = "auto"` to get the configuration from remote ClickHouse server. It will update itself on each `rollup-auto-interval` (1 minute by default) or once on startup if set to "0s".
 
 If you don't use a `GraphiteMergeTree` family engine, you can still use `rollup-conf = "auto"` by setting `rollup-auto-table="graphiteMergeTreeTable"` and get the proper config. In this case `graphiteMergeTreeTable` is a dummy table associated with proper [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration). The cases when you may need it:
 

doc/config.md

@@ -62,7 +62,7 @@ The rollup configuration is used for a proper  metrics pre-aggregation. It conta
 
 Historically, the way to define the config was `rollup-conf = "/path/to/the/conf/with/graphite_rollup.xml"`. The format is the same as [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration) scheme for ClickHouse server.
 
-For a quite long time it's recommended to use `rollup-conf = "auto"` to get the configuration from remote ClickHouse server. It will update itself each minute.
+For a quite long time it's recommended to use `rollup-conf = "auto"` to get the configuration from remote ClickHouse server. It will update itself on each `rollup-auto-interval` (1 minute by default) or once on startup if set to "0s".
 
 If you don't use a `GraphiteMergeTree` family engine, you can still use `rollup-conf = "auto"` by setting `rollup-auto-table="graphiteMergeTreeTable"` and get the proper config. In this case `graphiteMergeTreeTable` is a dummy table associated with proper [graphite_rollup](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/graphitemergetree/#rollup-configuration). The cases when you may need it:
 
@@ -205,6 +205,8 @@ It's possible to set multiple loggers. See `Config` description in [config.go](h
  rollup-conf = "auto"
  # custom table for 'rollup-conf=auto', useful for Distributed or MatView
  rollup-auto-table = ""
+ # rollup update interval for 'rollup-conf=auto'
+ rollup-auto-interval = "1m0s"
  # is used when none of rules match
  rollup-default-precision = 0
  # is used when none of rules match

helper/rollup/rollup.go

@@ -104,8 +104,10 @@ func (r *Rollup) updateWorker() {
 		// If we still have no rules - try every second to fetch them
 		if r.rules == nil {
 			time.Sleep(1 * time.Second)
-		} else {
+		} else if r.interval != 0 {
 			time.Sleep(r.interval)
+		} else {
+			break
 		}
 	}
 }