Merge branch 'master' into time-partioning

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     docker:
       # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.12.5
     working_directory: /go/src/github.com/improbable-eng/thanos
     environment:
       GO111MODULE: 'on'
@@ -62,7 +62,7 @@ jobs:
   publish_master:
     docker:
       # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.12.5
     working_directory: /go/src/github.com/improbable-eng/thanos
     environment:
       GOBIN: "/go/bin"
@@ -81,7 +81,7 @@ jobs:
   publish_release:
     docker:
       # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.12.5
     working_directory: /go/src/github.com/improbable-eng/thanos
     environment:
       GOBIN: "/go/bin"

.gitignore

@@ -1,6 +1,6 @@
 /prometheus
 /thanos
-vendor/
+/vendor/
 
 # Ignore minikube setup working dirs.
 kube/bin

CHANGELOG.md

@@ -14,6 +14,24 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 ### Added
 
 - [#1077](https://github.com/improbable-eng/thanos/pull/1077) Thanos Store gateway now supports time & duration based partitioning using `--min-time` & `--max-time` flags. Flags can be set to constant time in RFC3339 format or time duration relative to current time, such as -1.5d or 2h45m. Valid duration units are ms, s, m, h, d, w, y.
+- [#1094](https://github.com/improbable-eng/thanos/pull/1094) Allow configuring the response header timeout for the S3 client.
+
+### Changed
+
+- [#1066](https://github.com/improbable-eng/thanos/pull/1066) Upgrade Thanos ui to Prometheus v2.9.1.
+
+  Changes from the upstream:
+  * query:
+    - [ENHANCEMENT] Update moment.js and moment-timezone.js [PR #4679](https://github.com/prometheus/prometheus/pull/4679)
+    - [ENHANCEMENT] Support to query elements by a specific time [PR #4764](https://github.com/prometheus/prometheus/pull/4764)
+    - [ENHANCEMENT] Update to Bootstrap 4.1.3 [PR #5192](https://github.com/prometheus/prometheus/pull/5192)
+    - [BUGFIX] Limit number of merics in prometheus UI [PR #5139](https://github.com/prometheus/prometheus/pull/5139)
+    - [BUGFIX] Web interface Quality of Life improvements [PR #5201](https://github.com/prometheus/prometheus/pull/5201)
+  * rule:
+    - [ENHANCEMENT] Improve rule views by wrapping lines [PR #4702](https://github.com/prometheus/prometheus/pull/4702)
+    - [ENHANCEMENT] Show rule evaluation errors on rules page [PR #4457](https://github.com/prometheus/prometheus/pull/4457)
+
+- [#1156](https://github.com/improbable-eng/thanos/pull/1156) Moved CI and docker multistage to Golang 1.12.5 for latest mem alloc improvements. 
 
 ## [v0.4.0](https://github.com/improbable-eng/thanos/releases/tag/v0.4.0) - 2019.05.3
 

CONTRIBUTING.md

@@ -68,7 +68,7 @@ CI runs GCS and inmem tests only for now. Not having these variables will produc
 
 ## Dependency management
 
-The Thanos project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. This requires a working Go environment with version 1.11 or greater, git and [bzr](http://wiki.bazaar.canonical.com/Download) installed.
+The Thanos project uses [Go modules](https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more) to manage dependencies on external packages. This requires a working Go environment with version 1.11 or greater and git installed.
 
 To add or update a new dependency, use the `go get` command:
 

Dockerfile.multi-stage

@@ -1,11 +1,11 @@
-FROM golang:1.12-alpine3.9 as builder
+FROM golang:1.12.5-alpine3.9 as builder
 
 ADD . $GOPATH/src/github.com/improbable-eng/thanos
 WORKDIR $GOPATH/src/github.com/improbable-eng/thanos
 
 RUN apk update && apk upgrade && apk add --no-cache alpine-sdk
 
-RUN git update-index --refresh; make
+RUN git update-index --refresh; make build
 
 # -----------------------------------------------------------------------------
 

Makefile

@@ -1,12 +1,15 @@
 PREFIX            ?= $(shell pwd)
-DIRECTORIES       ?= $(shell find . -path './*' -prune -type d -not -path "./vendor")
+FILES_TO_FMT      ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print)
+
 DOCKER_IMAGE_NAME ?= thanos
 DOCKER_IMAGE_TAG  ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))-$(shell date +%Y-%m-%d)-$(shell git rev-parse --short HEAD)
 
 TMP_GOPATH        ?= /tmp/thanos-go
 GOBIN             ?= ${GOPATH}/bin
 GO111MODULE       ?= on
 export GO111MODULE
+GOPROXY           ?= https://proxy.golang.org
+export GOPROXY
 
 # Tools.
 EMBEDMD           ?= $(GOBIN)/embedmd-$(EMBEDMD_VERSION)
@@ -18,10 +21,10 @@ ERRCHECK_VERSION  ?= e14f8d59a22d460d56c5ee92507cd94c78fbf274
 LICHE             ?= $(GOBIN)/liche-$(LICHE_VERSION)
 LICHE_VERSION     ?= 2a2e6e56f6c615c17b2e116669c4cdb31b5453f3
 GOIMPORTS         ?= $(GOBIN)/goimports-$(GOIMPORTS_VERSION)
-GOIMPORTS_VERSION ?= 1c3d964395ce8f04f3b03b30aaed0b096c08c3c6
+GOIMPORTS_VERSION ?= 9d4d845e86f14303813298ede731a971dd65b593
 PROMU             ?= $(GOBIN)/promu-$(PROMU_VERSION)
-# v0.2.0
-PROMU_VERSION     ?= 264dc36af9ea3103255063497636bd5713e3e9c1
+# v0.4.0
+PROMU_VERSION     ?= afc7dfee7697527c08f2632f91b4c1369ed54798
 PROTOC            ?= $(GOBIN)/protoc-$(PROTOC_VERSION)
 PROTOC_VERSION    ?= 3.4.0
 # v0.55.3 This needs to match with version in netlify.toml
@@ -31,7 +34,6 @@ HUGO              ?= $(GOBIN)/hugo-$(HUGO_VERSION)
 GOBINDATA_VERSION ?= a9c83481b38ebb1c4eb8f0168fd4b10ca1d3c523
 GOBINDATA         ?= $(GOBIN)/go-bindata-$(GOBINDATA_VERSION)
 GIT               ?= $(shell which git)
-BZR               ?= $(shell which bzr)
 
 WEB_DIR           ?= website
 WEBSITE_BASE_URL  ?= https://thanos.io
@@ -104,7 +106,7 @@ assets: $(GOBINDATA)
 
 # build builds Thanos binary using `promu`.
 .PHONY: build
-build: check-git check-bzr go-mod-tidy $(PROMU)
+build: check-git  go-mod-tidy $(PROMU)
 	@echo ">> building binaries $(GOBIN)"
 	@$(PROMU) build --prefix $(PREFIX)
 
@@ -114,6 +116,11 @@ crossbuild: $(PROMU)
 	@echo ">> crossbuilding all binaries"
 	$(PROMU) crossbuild -v
 
+# deps ensures fresh go.mod and go.sum.
+.PHONY: deps
+deps:
+	@go mod tidy
+
 # docker builds docker with no tag.
 .PHONY: docker
 docker: build
@@ -157,11 +164,11 @@ errcheck: $(ERRCHECK)
 .PHONY: format
 format: $(GOIMPORTS)
 	@echo ">> formatting code"
-	@$(GOIMPORTS) -w $(DIRECTORIES)
+	@$(GOIMPORTS) -w $(FILES_TO_FMT)
 
 # proto generates golang files from Thanos proto files.
 .PHONY: proto
-proto: check-git check-bzr $(GOIMPORTS) $(PROTOC)
+proto: check-git  $(GOIMPORTS) $(PROTOC)
 	@go install ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast
 	@GOIMPORTS_BIN="$(GOIMPORTS)" PROTOC_BIN="$(PROTOC)" scripts/genproto.sh
 
@@ -183,7 +190,7 @@ tarballs-release: $(PROMU)
 
 # test runs all Thanos golang tests against each supported version of Prometheus.
 .PHONY: test
-test: check-git check-bzr test-deps
+test: check-git test-deps
 	@echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets"
 	THANOS_TEST_PROMETHEUS_VERSIONS="$(PROM_VERSIONS)" THANOS_TEST_ALERTMANAGER_PATH="alertmanager-$(ALERTMANAGER_VERSION)" go test $(shell go list ./... | grep -v /vendor/ | grep -v /benchmark/);
 
@@ -198,13 +205,13 @@ test-deps:
 
 # vet vets the code.
 .PHONY: vet
-vet: check-git check-bzr
+vet: check-git
 	@echo ">> vetting code"
 	@go vet ./...
 
 # go mod related
 .PHONY: go-mod-tidy
-go-mod-tidy: check-git check-bzr
+go-mod-tidy: check-git
 	@go mod tidy
 
 .PHONY: check-go-mod
@@ -220,13 +227,6 @@ else
 	@echo >&2 "No git binary found."; exit 1
 endif
 
-.PHONY: check-bzr
-check-bzr:
-ifneq ($(BZR),)
-	@test -x $(BZR) || (echo >&2 "No bzr exectuable binary found at $(BZR)."; exit 1)
-else
-	@echo >&2 "No bzr binary found."; exit 1
-endif
 
 .PHONY: web-pre-process
 web-pre-process:

README.md

@@ -30,8 +30,8 @@ Concretely the aims of the project are:
 
 ## Getting Started
 
-* **[Getting Started](docs/getting-started.md)**
-* [Design](docs/design.md)
+* **[Getting Started](https://thanos.io/getting-started.md/)**
+* [Design](https://thanos.io/design.md/)
 * [Prom Meetup Slides](https://www.slideshare.net/BartomiejPotka/thanos-global-durable-prometheus-monitoring)
 * [Introduction blog post](https://improbable.io/games/blog/thanos-prometheus-at-scale)
 * [Benchmarks](https://github.com/improbable-eng/thanos/tree/master/benchmark)
@@ -83,4 +83,4 @@ of the community. Here are ways to get in touch with the community:
 
 ## Maintainers
 
-See [MAINTAINERS.md](MAINTAINERS.md)
+See [MAINTAINERS.md](MAINTAINERS.md)

benchmark/cmd/thanosbench/main.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/prometheus/common/version"
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 	"k8s.io/client-go/tools/clientcmd"
 )
 

benchmark/cmd/thanosbench/resources.go

@@ -10,7 +10,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 	prom "github.com/prometheus/prometheus/config"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"

cmd/thanos/bucket.go

@@ -20,13 +20,13 @@ import (
 	"github.com/oklog/run"
 	"github.com/oklog/ulid"
 	"github.com/olekukonko/tablewriter"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/tsdb/labels"
 	"golang.org/x/text/language"
 	"golang.org/x/text/message"
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (

cmd/thanos/compact.go

@@ -21,11 +21,11 @@ import (
 	"github.com/improbable-eng/thanos/pkg/objstore/client"
 	"github.com/improbable-eng/thanos/pkg/runutil"
 	"github.com/oklog/run"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/tsdb"
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (

cmd/thanos/downsample.go

@@ -19,12 +19,12 @@ import (
 	"github.com/improbable-eng/thanos/pkg/runutil"
 	"github.com/oklog/run"
 	"github.com/oklog/ulid"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/tsdb"
 	"github.com/prometheus/tsdb/chunkenc"
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 func registerDownsample(m map[string]setupFunc, app *kingpin.Application, name string) {

cmd/thanos/flags.go

@@ -14,7 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 func regGRPCFlags(cmd *kingpin.CmdClause) (

cmd/thanos/main.go

@@ -27,7 +27,7 @@ import (
 	"github.com/improbable-eng/thanos/pkg/runutil"
 	"github.com/improbable-eng/thanos/pkg/tracing"
 	"github.com/oklog/run"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"

cmd/thanos/receive.go

@@ -18,14 +18,14 @@ import (
 	"github.com/improbable-eng/thanos/pkg/store"
 	"github.com/improbable-eng/thanos/pkg/store/storepb"
 	"github.com/oklog/run"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/storage/tsdb"
 	"github.com/prometheus/tsdb/labels"
 	"google.golang.org/grpc"
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 func registerReceive(m map[string]setupFunc, app *kingpin.Application, name string) {

cmd/thanos/rule.go

@@ -39,7 +39,7 @@ import (
 	"github.com/improbable-eng/thanos/pkg/tracing"
 	"github.com/improbable-eng/thanos/pkg/ui"
 	"github.com/oklog/run"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
@@ -53,7 +53,7 @@ import (
 	"github.com/prometheus/prometheus/util/strutil"
 	"github.com/prometheus/tsdb/labels"
 	"google.golang.org/grpc"
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 // registerRule registers a rule command.

cmd/thanos/sidecar.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
-	"github.com/hashicorp/go-version"
+	version "github.com/hashicorp/go-version"
 	"github.com/improbable-eng/thanos/pkg/block/metadata"
 	"github.com/improbable-eng/thanos/pkg/cluster"
 	"github.com/improbable-eng/thanos/pkg/component"