From 23294aa6c3f5a60683bdfc61e1994560f598d82a Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 4 Dec 2023 14:56:31 -0800 Subject: [PATCH] upgrade Prometheus version to latest main (#5697) Signed-off-by: Ben Ye --- go.mod | 18 +- go.sum | 61 +- .../golang_protobuf_extensions/v2/LICENSE | 201 ++++ .../golang_protobuf_extensions/v2/NOTICE | 1 + .../v2/pbutil/.gitignore | 1 + .../v2/pbutil/Makefile | 7 + .../v2/pbutil/decode.go | 81 ++ .../v2/pbutil/doc.go | 16 + .../v2/pbutil/encode.go | 49 + .../prometheus/common/config/http_config.go | 32 +- .../prometheus/common/expfmt/decode.go | 2 +- .../prometheus/common/expfmt/encode.go | 2 +- .../prometheus/prometheus/config/config.go | 103 ++- .../prometheus/discovery/dns/dns.go | 16 +- .../prometheus/discovery/manager.go | 2 +- .../prometheus/model/exemplar/exemplar.go | 15 + .../model/histogram/float_histogram.go | 360 ++++---- .../prometheus/model/histogram/generic.go | 175 +++- .../prometheus/model/histogram/histogram.go | 76 +- .../prometheus/model/histogram/test_utils.go | 52 ++ .../prometheus/model/labels/labels.go | 221 +---- .../prometheus/model/labels/labels_common.go | 235 +++++ .../model/labels/labels_stringlabels.go | 232 +---- .../prometheus/model/textparse/interface.go | 7 + .../model/textparse/openmetricsparse.go | 7 + .../prometheus/model/textparse/promparse.go | 7 + .../model/textparse/protobufparse.go | 61 +- .../prompb/io/prometheus/client/metrics.pb.go | 123 ++- .../prompb/io/prometheus/client/metrics.proto | 10 +- .../prometheus/prometheus/promql/engine.go | 92 +- .../prometheus/prometheus/promql/functions.go | 100 +- .../prometheus/promql/parser/ast.go | 1 - .../prometheus/promql/parser/functions.go | 26 +- .../promql/parser/generated_parser.y | 5 +- .../promql/parser/generated_parser.y.go | 187 ++-- .../prometheus/promql/parser/lex.go | 8 +- .../prometheus/promql/parser/parse.go | 9 +- .../prometheus/prometheus/promql/quantile.go | 93 +- .../prometheus/prometheus/promql/test.go | 20 +- .../prometheus/promql/testdata/functions.test | 110 +++ .../prometheus/prometheus/promql/value.go | 37 +- .../prometheus/prometheus/rules/alerting.go | 2 +- .../prometheus/prometheus/rules/group.go | 868 +++++++++++++++++ .../prometheus/prometheus/rules/manager.go | 873 +----------------- .../prometheus/prometheus/rules/rule.go | 64 ++ .../prometheus/prometheus/scrape/manager.go | 96 +- .../prometheus/prometheus/scrape/metrics.go | 307 ++++++ .../prometheus/prometheus/scrape/scrape.go | 420 ++++----- .../prometheus/prometheus/scrape/target.go | 36 +- .../prometheus/storage/interface.go | 18 +- .../prometheus/prometheus/storage/merge.go | 46 +- .../storage/remote/azuread/azuread.go | 8 +- .../prometheus/storage/remote/codec.go | 8 +- .../prometheus/normalize_label.go | 13 +- .../prometheus/normalize_name.go | 47 +- .../otlptranslator/prometheus/unit_to_ucum.go | 90 ++ .../prometheusremotewrite/helper.go | 137 +-- .../prometheusremotewrite/histograms.go | 82 +- .../prometheusremotewrite/metrics_to_prw.go | 4 +- .../number_data_points.go | 4 +- .../prometheus/storage/remote/read_handler.go | 7 +- .../prometheus/storage/remote/storage.go | 5 +- .../prometheus/storage/remote/write.go | 10 + .../storage/remote/write_handler.go | 16 +- .../prometheus/template/template.go | 2 +- .../prometheus/prometheus/tsdb/block.go | 18 +- .../prometheus/tsdb/chunkenc/chunk.go | 38 +- .../tsdb/chunkenc/float_histogram.go | 8 +- .../prometheus/tsdb/chunkenc/histogram.go | 8 +- .../tsdb/chunkenc/histogram_meta.go | 2 +- .../prometheus/tsdb/chunkenc/varbit.go | 7 +- .../prometheus/tsdb/chunkenc/xor.go | 2 +- .../prometheus/tsdb/chunks/chunks.go | 71 +- .../prometheus/tsdb/chunks/head_chunks.go | 56 +- .../prometheus/prometheus/tsdb/compact.go | 11 +- .../prometheus/prometheus/tsdb/db.go | 162 ++-- .../prometheus/tsdb/encoding/encoding.go | 5 +- .../prometheus/tsdb/errors/errors.go | 5 +- .../prometheus/prometheus/tsdb/exemplar.go | 19 +- .../prometheus/tsdb/fileutil/mmap.go | 9 +- .../tsdb/fileutil/preallocate_linux.go | 9 +- .../prometheus/prometheus/tsdb/head.go | 262 ++++-- .../prometheus/prometheus/tsdb/head_append.go | 127 +-- .../prometheus/prometheus/tsdb/head_read.go | 107 +-- .../prometheus/prometheus/tsdb/head_wal.go | 11 +- .../prometheus/prometheus/tsdb/index/index.go | 134 +-- .../prometheus/tsdb/index/postings.go | 4 +- .../prometheus/tsdb/index/postingsstats.go | 11 +- .../prometheus/prometheus/tsdb/ooo_head.go | 26 +- .../prometheus/tsdb/ooo_head_read.go | 102 +- .../prometheus/tsdb/ooo_isolation.go | 79 ++ .../prometheus/prometheus/tsdb/querier.go | 259 +++++- .../prometheus/tsdb/record/record.go | 41 +- .../prometheus/prometheus/tsdb/repair.go | 3 +- .../prometheus/tsdb/tombstones/tombstones.go | 14 +- .../prometheus/tsdb/tsdbutil/dir_locker.go | 4 +- .../prometheus/tsdb/tsdbutil/histogram.go | 10 + .../prometheus/prometheus/tsdb/wal.go | 15 +- .../prometheus/tsdb/wlog/checkpoint.go | 46 +- .../prometheus/tsdb/wlog/live_reader.go | 12 +- .../prometheus/prometheus/tsdb/wlog/reader.go | 17 +- .../prometheus/tsdb/wlog/watcher.go | 30 +- .../prometheus/prometheus/tsdb/wlog/wlog.go | 54 +- .../util/annotations/annotations.go | 27 +- .../prometheus/util/testutil/context.go | 8 +- .../prometheus/util/testutil/directory.go | 2 +- .../prometheus/prometheus/web/api/v1/api.go | 97 +- .../thanos-io/thanos/pkg/block/index.go | 3 +- .../pkg/block/indexheader/binary_reader.go | 2 +- .../pkg/compact/downsample/downsample.go | 3 +- .../thanos/pkg/errutil/multierror.go | 2 +- .../thanos-io/thanos/pkg/httpconfig/http.go | 3 +- .../thanos-io/thanos/pkg/runutil/runutil.go | 3 +- .../thanos-io/thanos/pkg/store/bucket.go | 4 +- .../thanos/pkg/store/cache/filter_cache.go | 27 +- .../collector/featuregate/LICENSE | 202 ++++ .../collector/featuregate/Makefile | 1 + .../collector/featuregate/README.md | 73 ++ .../collector/featuregate/flag.go | 66 ++ .../collector/featuregate/gate.go | 64 ++ .../collector/featuregate/registry.go | 149 +++ .../collector/featuregate/stage.go | 55 ++ .../collector/pdata/pmetric/metrics.go | 9 + .../tools/go/internal/packagesdriver/sizes.go | 15 +- .../golang.org/x/tools/go/packages/golist.go | 74 -- .../x/tools/go/packages/golist_overlay.go | 492 ---------- .../x/tools/go/packages/packages.go | 169 ++-- .../x/tools/go/types/objectpath/objectpath.go | 117 +-- .../x/tools/internal/gocommand/invoke.go | 24 +- .../internal/typesinternal/objectpath.go | 24 - vendor/modules.txt | 24 +- 131 files changed, 5673 insertions(+), 3888 deletions(-) create mode 100644 vendor/github.com/matttproud/golang_protobuf_extensions/v2/LICENSE create mode 100644 vendor/github.com/matttproud/golang_protobuf_extensions/v2/NOTICE create mode 100644 vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/.gitignore create mode 100644 vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/Makefile create mode 100644 vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/decode.go create mode 100644 vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/doc.go create mode 100644 vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/encode.go create mode 100644 vendor/github.com/prometheus/prometheus/model/histogram/test_utils.go create mode 100644 vendor/github.com/prometheus/prometheus/model/labels/labels_common.go create mode 100644 vendor/github.com/prometheus/prometheus/rules/group.go create mode 100644 vendor/github.com/prometheus/prometheus/rules/rule.go create mode 100644 vendor/github.com/prometheus/prometheus/scrape/metrics.go create mode 100644 vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/unit_to_ucum.go create mode 100644 vendor/github.com/prometheus/prometheus/tsdb/ooo_isolation.go create mode 100644 vendor/go.opentelemetry.io/collector/featuregate/LICENSE create mode 100644 vendor/go.opentelemetry.io/collector/featuregate/Makefile create mode 100644 vendor/go.opentelemetry.io/collector/featuregate/README.md create mode 100644 vendor/go.opentelemetry.io/collector/featuregate/flag.go create mode 100644 vendor/go.opentelemetry.io/collector/featuregate/gate.go create mode 100644 vendor/go.opentelemetry.io/collector/featuregate/registry.go create mode 100644 vendor/go.opentelemetry.io/collector/featuregate/stage.go delete mode 100644 vendor/golang.org/x/tools/internal/typesinternal/objectpath.go diff --git a/go.mod b/go.mod index 5bfbe71179..9ab066a70f 100644 --- a/go.mod +++ b/go.mod @@ -44,16 +44,16 @@ require ( github.com/prometheus/alertmanager v0.26.0 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 - github.com/prometheus/common v0.44.0 + github.com/prometheus/common v0.45.0 // Prometheus maps version 2.x.y to tags v0.x.y. - github.com/prometheus/prometheus v0.48.0 + github.com/prometheus/prometheus v0.48.1-0.20231201222638-e4ec263bcc11 github.com/segmentio/fasthash v1.0.3 github.com/sony/gobreaker v0.5.0 github.com/spf13/afero v1.9.5 github.com/stretchr/testify v1.8.4 github.com/thanos-io/objstore v0.0.0-20231123170144-bffedaa58acb github.com/thanos-io/promql-engine v0.0.0-20231127105941-257543af55e8 - github.com/thanos-io/thanos v0.32.5-0.20231127170340-8ffb9da1383e + github.com/thanos-io/thanos v0.32.5-0.20231204192512-28407d61e72d github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d go.etcd.io/etcd/api/v3 v3.5.10 @@ -80,7 +80,7 @@ require ( github.com/VictoriaMetrics/fastcache v1.12.1 github.com/cespare/xxhash/v2 v2.2.0 github.com/google/go-cmp v0.6.0 - golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa google.golang.org/protobuf v1.31.0 ) @@ -171,6 +171,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/miekg/dns v1.1.56 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -203,8 +204,9 @@ require ( github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 // indirect go.mongodb.org/mongo-driver v1.12.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 // indirect - go.opentelemetry.io/collector/semconv v0.87.0 // indirect + go.opentelemetry.io/collector/featuregate v0.77.0 // indirect + go.opentelemetry.io/collector/pdata v1.0.0-rcv0017 // indirect + go.opentelemetry.io/collector/semconv v0.88.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/contrib/propagators/autoprop v0.38.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.13.0 // indirect @@ -218,11 +220,11 @@ require ( go4.org/intern v0.0.0-20230525184215-6c62f75575cb // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect golang.org/x/crypto v0.15.0 // indirect - golang.org/x/mod v0.13.0 // indirect + golang.org/x/mod v0.14.0 // indirect golang.org/x/oauth2 v0.14.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/tools v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gonum.org/v1/gonum v0.12.0 // indirect google.golang.org/api v0.150.0 // indirect diff --git a/go.sum b/go.sum index 5c676dcd00..cb7a132ec9 100644 --- a/go.sum +++ b/go.sum @@ -622,6 +622,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Code-Hex/go-generics-cache v1.3.1 h1:i8rLwyhoyhaerr7JpjtYjJZUcCbWOdiYO3fZXLiEC4g= +github.com/Code-Hex/go-generics-cache v1.3.1/go.mod h1:qxcC9kRVrct9rHeiYpFWSoW1vxyillCVzX13KZG8dl4= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= @@ -900,8 +902,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= -github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= +github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= @@ -956,8 +958,9 @@ github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -1243,8 +1246,8 @@ github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linode/linodego v1.23.0 h1:s0ReCZtuN9Z1IoUN9w1RLeYO1dMZUGPwOQ/IBFsBHtU= -github.com/linode/linodego v1.23.0/go.mod h1:0U7wj/UQOqBNbKv1FYTXiBUXueR8DY4HvIotwE0ENgg= +github.com/linode/linodego v1.24.0 h1:zO+bMdTE6wPccqP7QIkbxAfACX7DjSX6DW9JE/qOKDQ= +github.com/linode/linodego v1.24.0/go.mod h1:cq/ty5BCEQnsO6OjMqD7Q03KCCyB8CNM5E3MNg0LV6M= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= @@ -1278,6 +1281,8 @@ github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -1403,8 +1408,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdDeppTwGX4= github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/exporter-toolkit v0.8.2/go.mod h1:00shzmJL7KxcsabLWcONwpyNEuWhREOnFqZW7vadFS0= @@ -1419,8 +1424,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/prometheus/prometheus v0.48.0 h1:yrBloImGQ7je4h8M10ujGh4R6oxYQJQKlMuETwNskGk= -github.com/prometheus/prometheus v0.48.0/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g= +github.com/prometheus/prometheus v0.48.1-0.20231201222638-e4ec263bcc11 h1:3wogiAxGft1+Syu99dswxhbWpR4IjLFKQgKxU84wrag= +github.com/prometheus/prometheus v0.48.1-0.20231201222638-e4ec263bcc11/go.mod h1:hCcxMXhfC04Ua9hQi2j43+EGMQhrRNvH8x0LteAnF1I= github.com/redis/rueidis v1.0.14-go1.18 h1:dGir5z8w8X1ex7JWO/Zx2FMBrZgQ8Yjm+lw9fPLSNGw= github.com/redis/rueidis v1.0.14-go1.18/go.mod h1:HGekzV3HbmzFmRK6j0xic8Z9119+ECoGMjeN1TV1NYU= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -1504,8 +1509,8 @@ github.com/thanos-io/objstore v0.0.0-20231123170144-bffedaa58acb h1:3s/a99MWpt5Z github.com/thanos-io/objstore v0.0.0-20231123170144-bffedaa58acb/go.mod h1:RMvJQnpB4QQiYGg1gF8mnPJg6IkIPY28Buh8f6b+F0c= github.com/thanos-io/promql-engine v0.0.0-20231127105941-257543af55e8 h1:KX57eKPq3yzX7ENyd0+fOfPb6v9tjOCLRT8/9waWs/w= github.com/thanos-io/promql-engine v0.0.0-20231127105941-257543af55e8/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q= -github.com/thanos-io/thanos v0.32.5-0.20231127170340-8ffb9da1383e h1:5srK0HsmKwUlH7KksjIJwfv7v/jSwjOOhM76rzoFuCY= -github.com/thanos-io/thanos v0.32.5-0.20231127170340-8ffb9da1383e/go.mod h1:DzjNla690uSQazyHrLGK6FeOn3VfV2ReA57Pi7SgZ+8= +github.com/thanos-io/thanos v0.32.5-0.20231204192512-28407d61e72d h1:GJhhQxnEENOF0ZgHfK2WAC0qwQhAnexld0Ujhe0xgrY= +github.com/thanos-io/thanos v0.32.5-0.20231204192512-28407d61e72d/go.mod h1:tADvTBUwVH/yjWymDztTV8/bErQVIGFlHIZqXc07QQo= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -1567,10 +1572,12 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 h1:qCPXSQCoD3qeWFb1RuIks8fw9Atxpk78bmtVdi15KhE= -go.opentelemetry.io/collector/pdata v1.0.0-rcv0016/go.mod h1:OdN0alYOlYhHXu6BDlGehrZWgtBuiDsz/rlNeJeXiNg= -go.opentelemetry.io/collector/semconv v0.87.0 h1:BsG1jdLLRCBRlvUujk4QA86af7r/ZXnizczQpEs/gg8= -go.opentelemetry.io/collector/semconv v0.87.0/go.mod h1:j/8THcqVxFna1FpvA2zYIsUperEtOaRaqoLYIN4doWw= +go.opentelemetry.io/collector/featuregate v0.77.0 h1:m1/IzaXoQh6SgF6CM80vrBOCf5zSJ2GVISfA27fYzGU= +go.opentelemetry.io/collector/featuregate v0.77.0/go.mod h1:/kVAsGUCyJXIDSgHftCN63QiwAEVHRLX2Kh/S+dqgHY= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0017 h1:AgALhc2VenoA5l1DvTdg7mkzaBGqoTSuMkAtjsttBFo= +go.opentelemetry.io/collector/pdata v1.0.0-rcv0017/go.mod h1:Rv9fOclA5AtM/JGm0d4jBOIAo1+jBA13UT5Bx0ovXi4= +go.opentelemetry.io/collector/semconv v0.88.0 h1:8TVP4hYaUC87S6CCLKNoSxsUE0ChldE4vqotvNHHUnE= +go.opentelemetry.io/collector/semconv v0.88.0/go.mod h1:j/8THcqVxFna1FpvA2zYIsUperEtOaRaqoLYIN4doWw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= go.opentelemetry.io/contrib/propagators/autoprop v0.38.0 h1:WZwiLCwOL0XW/6TVT7LTtdRDveoHZ6q3wL+0iYsBcdE= @@ -1658,8 +1665,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1701,8 +1708,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2030,8 +2037,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2323,12 +2330,12 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions/v2/LICENSE b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions/v2/NOTICE b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/NOTICE new file mode 100644 index 0000000000..5d8cb5b72e --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/NOTICE @@ -0,0 +1 @@ +Copyright 2012 Matt T. Proud (matt.proud@gmail.com) diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/.gitignore b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/.gitignore new file mode 100644 index 0000000000..e16fb946bb --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/.gitignore @@ -0,0 +1 @@ +cover.dat diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/Makefile b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/Makefile new file mode 100644 index 0000000000..81be214370 --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/Makefile @@ -0,0 +1,7 @@ +all: + +cover: + go test -cover -v -coverprofile=cover.dat ./... + go tool cover -func cover.dat + +.PHONY: cover diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/decode.go b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/decode.go new file mode 100644 index 0000000000..7c08e564f1 --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/decode.go @@ -0,0 +1,81 @@ +// Copyright 2013 Matt T. Proud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pbutil + +import ( + "encoding/binary" + "errors" + "io" + + "google.golang.org/protobuf/proto" +) + +// TODO: Give error package name prefix in next minor release. +var errInvalidVarint = errors.New("invalid varint32 encountered") + +// ReadDelimited decodes a message from the provided length-delimited stream, +// where the length is encoded as 32-bit varint prefix to the message body. +// It returns the total number of bytes read and any applicable error. This is +// roughly equivalent to the companion Java API's +// MessageLite#parseDelimitedFrom. As per the reader contract, this function +// calls r.Read repeatedly as required until exactly one message including its +// prefix is read and decoded (or an error has occurred). The function never +// reads more bytes from the stream than required. The function never returns +// an error if a message has been read and decoded correctly, even if the end +// of the stream has been reached in doing so. In that case, any subsequent +// calls return (0, io.EOF). +func ReadDelimited(r io.Reader, m proto.Message) (n int, err error) { + // TODO: Consider allowing the caller to specify a decode buffer in the + // next major version. + + // TODO: Consider using error wrapping to annotate error state in pass- + // through cases in the next minor version. + + // Per AbstractParser#parsePartialDelimitedFrom with + // CodedInputStream#readRawVarint32. + var headerBuf [binary.MaxVarintLen32]byte + var bytesRead, varIntBytes int + var messageLength uint64 + for varIntBytes == 0 { // i.e. no varint has been decoded yet. + if bytesRead >= len(headerBuf) { + return bytesRead, errInvalidVarint + } + // We have to read byte by byte here to avoid reading more bytes + // than required. Each read byte is appended to what we have + // read before. + newBytesRead, err := r.Read(headerBuf[bytesRead : bytesRead+1]) + if newBytesRead == 0 { + if err != nil { + return bytesRead, err + } + // A Reader should not return (0, nil); but if it does, it should + // be treated as no-op according to the Reader contract. + continue + } + bytesRead += newBytesRead + // Now present everything read so far to the varint decoder and + // see if a varint can be decoded already. + messageLength, varIntBytes = binary.Uvarint(headerBuf[:bytesRead]) + } + + messageBuf := make([]byte, messageLength) + newBytesRead, err := io.ReadFull(r, messageBuf) + bytesRead += newBytesRead + if err != nil { + return bytesRead, err + } + + return bytesRead, proto.Unmarshal(messageBuf, m) +} diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/doc.go b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/doc.go new file mode 100644 index 0000000000..c318385cbe --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/doc.go @@ -0,0 +1,16 @@ +// Copyright 2013 Matt T. Proud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pbutil provides record length-delimited Protocol Buffer streaming. +package pbutil diff --git a/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/encode.go b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/encode.go new file mode 100644 index 0000000000..e58dd9d297 --- /dev/null +++ b/vendor/github.com/matttproud/golang_protobuf_extensions/v2/pbutil/encode.go @@ -0,0 +1,49 @@ +// Copyright 2013 Matt T. Proud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pbutil + +import ( + "encoding/binary" + "io" + + "google.golang.org/protobuf/proto" +) + +// WriteDelimited encodes and dumps a message to the provided writer prefixed +// with a 32-bit varint indicating the length of the encoded message, producing +// a length-delimited record stream, which can be used to chain together +// encoded messages of the same type together in a file. It returns the total +// number of bytes written and any applicable error. This is roughly +// equivalent to the companion Java API's MessageLite#writeDelimitedTo. +func WriteDelimited(w io.Writer, m proto.Message) (n int, err error) { + // TODO: Consider allowing the caller to specify an encode buffer in the + // next major version. + + buffer, err := proto.Marshal(m) + if err != nil { + return 0, err + } + + var buf [binary.MaxVarintLen32]byte + encodedLength := binary.PutUvarint(buf[:], uint64(len(buffer))) + + sync, err := w.Write(buf[:encodedLength]) + if err != nil { + return sync, err + } + + n, err = w.Write(buffer) + return n + sync, err +} diff --git a/vendor/github.com/prometheus/common/config/http_config.go b/vendor/github.com/prometheus/common/config/http_config.go index 37aa966748..4763549b8f 100644 --- a/vendor/github.com/prometheus/common/config/http_config.go +++ b/vendor/github.com/prometheus/common/config/http_config.go @@ -129,6 +129,7 @@ func (tv *TLSVersion) String() string { // BasicAuth contains basic HTTP authentication credentials. type BasicAuth struct { Username string `yaml:"username" json:"username"` + UsernameFile string `yaml:"username_file,omitempty" json:"username_file,omitempty"` Password Secret `yaml:"password,omitempty" json:"password,omitempty"` PasswordFile string `yaml:"password_file,omitempty" json:"password_file,omitempty"` } @@ -139,6 +140,7 @@ func (a *BasicAuth) SetDirectory(dir string) { return } a.PasswordFile = JoinDir(dir, a.PasswordFile) + a.UsernameFile = JoinDir(dir, a.UsernameFile) } // Authorization contains HTTP authorization credentials. @@ -334,6 +336,9 @@ func (c *HTTPClientConfig) Validate() error { if (c.BasicAuth != nil || c.OAuth2 != nil) && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) { return fmt.Errorf("at most one of basic_auth, oauth2, bearer_token & bearer_token_file must be configured") } + if c.BasicAuth != nil && (string(c.BasicAuth.Username) != "" && c.BasicAuth.UsernameFile != "") { + return fmt.Errorf("at most one of basic_auth username & username_file must be configured") + } if c.BasicAuth != nil && (string(c.BasicAuth.Password) != "" && c.BasicAuth.PasswordFile != "") { return fmt.Errorf("at most one of basic_auth password & password_file must be configured") } @@ -555,7 +560,7 @@ func NewRoundTripperFromConfig(cfg HTTPClientConfig, name string, optFuncs ...HT } if cfg.BasicAuth != nil { - rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.PasswordFile, rt) + rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.UsernameFile, cfg.BasicAuth.PasswordFile, rt) } if cfg.OAuth2 != nil { @@ -645,30 +650,43 @@ func (rt *authorizationCredentialsFileRoundTripper) CloseIdleConnections() { type basicAuthRoundTripper struct { username string password Secret + usernameFile string passwordFile string rt http.RoundTripper } // NewBasicAuthRoundTripper will apply a BASIC auth authorization header to a request unless it has // already been set. -func NewBasicAuthRoundTripper(username string, password Secret, passwordFile string, rt http.RoundTripper) http.RoundTripper { - return &basicAuthRoundTripper{username, password, passwordFile, rt} +func NewBasicAuthRoundTripper(username string, password Secret, usernameFile, passwordFile string, rt http.RoundTripper) http.RoundTripper { + return &basicAuthRoundTripper{username, password, usernameFile, passwordFile, rt} } func (rt *basicAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + var username string + var password string if len(req.Header.Get("Authorization")) != 0 { return rt.rt.RoundTrip(req) } - req = cloneRequest(req) + if rt.usernameFile != "" { + usernameBytes, err := os.ReadFile(rt.usernameFile) + if err != nil { + return nil, fmt.Errorf("unable to read basic auth username file %s: %s", rt.usernameFile, err) + } + username = strings.TrimSpace(string(usernameBytes)) + } else { + username = rt.username + } if rt.passwordFile != "" { - bs, err := os.ReadFile(rt.passwordFile) + passwordBytes, err := os.ReadFile(rt.passwordFile) if err != nil { return nil, fmt.Errorf("unable to read basic auth password file %s: %s", rt.passwordFile, err) } - req.SetBasicAuth(rt.username, strings.TrimSpace(string(bs))) + password = strings.TrimSpace(string(passwordBytes)) } else { - req.SetBasicAuth(rt.username, strings.TrimSpace(string(rt.password))) + password = string(rt.password) } + req = cloneRequest(req) + req.SetBasicAuth(username, password) return rt.rt.RoundTrip(req) } diff --git a/vendor/github.com/prometheus/common/expfmt/decode.go b/vendor/github.com/prometheus/common/expfmt/decode.go index 9063978151..0ca86a3dc7 100644 --- a/vendor/github.com/prometheus/common/expfmt/decode.go +++ b/vendor/github.com/prometheus/common/expfmt/decode.go @@ -22,7 +22,7 @@ import ( dto "github.com/prometheus/client_model/go" - "github.com/matttproud/golang_protobuf_extensions/pbutil" + "github.com/matttproud/golang_protobuf_extensions/v2/pbutil" "github.com/prometheus/common/model" ) diff --git a/vendor/github.com/prometheus/common/expfmt/encode.go b/vendor/github.com/prometheus/common/expfmt/encode.go index 7f611ffaad..ca21406000 100644 --- a/vendor/github.com/prometheus/common/expfmt/encode.go +++ b/vendor/github.com/prometheus/common/expfmt/encode.go @@ -18,7 +18,7 @@ import ( "io" "net/http" - "github.com/matttproud/golang_protobuf_extensions/pbutil" + "github.com/matttproud/golang_protobuf_extensions/v2/pbutil" "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" "google.golang.org/protobuf/encoding/prototext" diff --git a/vendor/github.com/prometheus/prometheus/config/config.go b/vendor/github.com/prometheus/prometheus/config/config.go index f0b359d8e1..b832ac9a17 100644 --- a/vendor/github.com/prometheus/prometheus/config/config.go +++ b/vendor/github.com/prometheus/prometheus/config/config.go @@ -19,6 +19,7 @@ import ( "net/url" "os" "path/filepath" + "sort" "strings" "time" @@ -143,18 +144,21 @@ var ( ScrapeInterval: model.Duration(1 * time.Minute), ScrapeTimeout: model.Duration(10 * time.Second), EvaluationInterval: model.Duration(1 * time.Minute), + // When native histogram feature flag is enabled, ScrapeProtocols default + // changes to DefaultNativeHistogramScrapeProtocols. + ScrapeProtocols: DefaultScrapeProtocols, } // DefaultScrapeConfig is the default scrape configuration. DefaultScrapeConfig = ScrapeConfig{ - // ScrapeTimeout and ScrapeInterval default to the configured - // globals. + // ScrapeTimeout, ScrapeInterval and ScrapeProtocols default to the configured globals. ScrapeClassicHistograms: false, MetricsPath: "/metrics", Scheme: "http", HonorLabels: false, HonorTimestamps: true, HTTPClientConfig: config.DefaultHTTPClientConfig, + EnableCompression: true, } // DefaultAlertmanagerConfig is the default alertmanager configuration. @@ -260,7 +264,7 @@ func (c Config) String() string { return string(b) } -// ScrapeConfigs returns the scrape configurations. +// GetScrapeConfigs returns the scrape configurations. func (c *Config) GetScrapeConfigs() ([]*ScrapeConfig, error) { scfgs := make([]*ScrapeConfig, len(c.ScrapeConfigs)) @@ -385,6 +389,11 @@ type GlobalConfig struct { ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"` // The default timeout when scraping targets. ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"` + // The protocols to negotiate during a scrape. It tells clients what + // protocol are accepted by Prometheus and with what weight (most wanted is first). + // Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, + // OpenMetricsText1.0.0, PrometheusText0.0.4. + ScrapeProtocols []ScrapeProtocol `yaml:"scrape_protocols,omitempty"` // How frequently to evaluate rules by default. EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"` // File to which PromQL queries are logged. @@ -414,6 +423,68 @@ type GlobalConfig struct { KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"` } +// ScrapeProtocol represents supported protocol for scraping metrics. +type ScrapeProtocol string + +// Validate returns error if given scrape protocol is not supported. +func (s ScrapeProtocol) Validate() error { + if _, ok := ScrapeProtocolsHeaders[s]; !ok { + return fmt.Errorf("unknown scrape protocol %v, supported: %v", + s, func() (ret []string) { + for k := range ScrapeProtocolsHeaders { + ret = append(ret, string(k)) + } + sort.Strings(ret) + return ret + }()) + } + return nil +} + +var ( + PrometheusProto ScrapeProtocol = "PrometheusProto" + PrometheusText0_0_4 ScrapeProtocol = "PrometheusText0.0.4" + OpenMetricsText0_0_1 ScrapeProtocol = "OpenMetricsText0.0.1" + OpenMetricsText1_0_0 ScrapeProtocol = "OpenMetricsText1.0.0" + + ScrapeProtocolsHeaders = map[ScrapeProtocol]string{ + PrometheusProto: "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited", + PrometheusText0_0_4: "text/plain;version=0.0.4", + OpenMetricsText0_0_1: "application/openmetrics-text;version=0.0.1", + OpenMetricsText1_0_0: "application/openmetrics-text;version=1.0.0", + } + + DefaultScrapeProtocols = []ScrapeProtocol{ + OpenMetricsText1_0_0, + OpenMetricsText0_0_1, + PrometheusText0_0_4, + } + DefaultNativeHistogramScrapeProtocols = []ScrapeProtocol{ + PrometheusProto, + OpenMetricsText1_0_0, + OpenMetricsText0_0_1, + PrometheusText0_0_4, + } +) + +// validateAcceptScrapeProtocols return errors if we see problems with accept scrape protocols option. +func validateAcceptScrapeProtocols(sps []ScrapeProtocol) error { + if len(sps) == 0 { + return errors.New("scrape_protocols cannot be empty") + } + dups := map[string]struct{}{} + for _, sp := range sps { + if _, ok := dups[strings.ToLower(string(sp))]; ok { + return fmt.Errorf("duplicated protocol in scrape_protocols, got %v", sps) + } + if err := sp.Validate(); err != nil { + return fmt.Errorf("scrape_protocols: %w", err) + } + dups[strings.ToLower(string(sp))] = struct{}{} + } + return nil +} + // SetDirectory joins any relative file paths with dir. func (c *GlobalConfig) SetDirectory(dir string) { c.QueryLogFile = config.JoinDir(dir, c.QueryLogFile) @@ -459,6 +530,14 @@ func (c *GlobalConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if gc.EvaluationInterval == 0 { gc.EvaluationInterval = DefaultGlobalConfig.EvaluationInterval } + + if gc.ScrapeProtocols == nil { + gc.ScrapeProtocols = DefaultGlobalConfig.ScrapeProtocols + } + if err := validateAcceptScrapeProtocols(gc.ScrapeProtocols); err != nil { + return fmt.Errorf("%w for global config", err) + } + *c = *gc return nil } @@ -469,7 +548,8 @@ func (c *GlobalConfig) isZero() bool { c.ScrapeInterval == 0 && c.ScrapeTimeout == 0 && c.EvaluationInterval == 0 && - c.QueryLogFile == "" + c.QueryLogFile == "" && + c.ScrapeProtocols == nil } type ScrapeConfigs struct { @@ -492,12 +572,19 @@ type ScrapeConfig struct { ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"` // The timeout for scraping targets of this config. ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"` + // The protocols to negotiate during a scrape. It tells clients what + // protocol are accepted by Prometheus and with what preference (most wanted is first). + // Supported values (case sensitive): PrometheusProto, OpenMetricsText0.0.1, + // OpenMetricsText1.0.0, PrometheusText0.0.4. + ScrapeProtocols []ScrapeProtocol `yaml:"scrape_protocols,omitempty"` // Whether to scrape a classic histogram that is also exposed as a native histogram. ScrapeClassicHistograms bool `yaml:"scrape_classic_histograms,omitempty"` // The HTTP resource path on which to fetch metrics from targets. MetricsPath string `yaml:"metrics_path,omitempty"` // The URL scheme with which to fetch metrics from targets. Scheme string `yaml:"scheme,omitempty"` + // Indicator whether to request compressed response from the target. + EnableCompression bool `yaml:"enable_compression"` // An uncompressed response body larger than this many bytes will cause the // scrape to fail. 0 means no limit. BodySizeLimit units.Base2Bytes `yaml:"body_size_limit,omitempty"` @@ -579,6 +666,7 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// Validate validates scrape config, but also fills relevant default values from global config if needed. func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error { if c == nil { return errors.New("empty or null scrape config section") @@ -620,6 +708,13 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error { c.KeepDroppedTargets = globalConfig.KeepDroppedTargets } + if c.ScrapeProtocols == nil { + c.ScrapeProtocols = globalConfig.ScrapeProtocols + } + if err := validateAcceptScrapeProtocols(c.ScrapeProtocols); err != nil { + return fmt.Errorf("%w for scrape config with job name %q", err, c.JobName) + } + return nil } diff --git a/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go b/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go index 96e07254f0..4838a89547 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go +++ b/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go @@ -42,6 +42,8 @@ const ( dnsSrvRecordPortLabel = dnsSrvRecordPrefix + "port" dnsMxRecordPrefix = model.MetaLabelPrefix + "dns_mx_record_" dnsMxRecordTargetLabel = dnsMxRecordPrefix + "target" + dnsNsRecordPrefix = model.MetaLabelPrefix + "dns_ns_record_" + dnsNsRecordTargetLabel = dnsNsRecordPrefix + "target" // Constants for instrumentation. namespace = "prometheus" @@ -102,7 +104,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { } switch strings.ToUpper(c.Type) { case "SRV": - case "A", "AAAA", "MX": + case "A", "AAAA", "MX", "NS": if c.Port == 0 { return errors.New("a port is required in DNS-SD configs for all record types except SRV") } @@ -140,6 +142,8 @@ func NewDiscovery(conf SDConfig, logger log.Logger) *Discovery { qtype = dns.TypeSRV case "MX": qtype = dns.TypeMX + case "NS": + qtype = dns.TypeNS } d := &Discovery{ names: conf.Names, @@ -199,7 +203,7 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ } for _, record := range response.Answer { - var target, dnsSrvRecordTarget, dnsSrvRecordPort, dnsMxRecordTarget model.LabelValue + var target, dnsSrvRecordTarget, dnsSrvRecordPort, dnsMxRecordTarget, dnsNsRecordTarget model.LabelValue switch addr := record.(type) { case *dns.SRV: @@ -217,6 +221,13 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ addr.Mx = strings.TrimRight(addr.Mx, ".") target = hostPort(addr.Mx, d.port) + case *dns.NS: + dnsNsRecordTarget = model.LabelValue(addr.Ns) + + // Remove the final dot from rooted DNS names to make them look more usual. + addr.Ns = strings.TrimRight(addr.Ns, ".") + + target = hostPort(addr.Ns, d.port) case *dns.A: target = hostPort(addr.A.String(), d.port) case *dns.AAAA: @@ -234,6 +245,7 @@ func (d *Discovery) refreshOne(ctx context.Context, name string, ch chan<- *targ dnsSrvRecordTargetLabel: dnsSrvRecordTarget, dnsSrvRecordPortLabel: dnsSrvRecordPort, dnsMxRecordTargetLabel: dnsMxRecordTarget, + dnsNsRecordTargetLabel: dnsNsRecordTarget, }) } diff --git a/vendor/github.com/prometheus/prometheus/discovery/manager.go b/vendor/github.com/prometheus/prometheus/discovery/manager.go index 4d6027691f..86439d2c95 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/manager.go +++ b/vendor/github.com/prometheus/prometheus/discovery/manager.go @@ -92,7 +92,7 @@ type Provider struct { newSubs map[string]struct{} } -// Discoverer return the Discoverer of the provider +// Discoverer return the Discoverer of the provider. func (p *Provider) Discoverer() Discoverer { return p.d } diff --git a/vendor/github.com/prometheus/prometheus/model/exemplar/exemplar.go b/vendor/github.com/prometheus/prometheus/model/exemplar/exemplar.go index 2e39cf6892..08f55374ef 100644 --- a/vendor/github.com/prometheus/prometheus/model/exemplar/exemplar.go +++ b/vendor/github.com/prometheus/prometheus/model/exemplar/exemplar.go @@ -48,3 +48,18 @@ func (e Exemplar) Equals(e2 Exemplar) bool { return e.Value == e2.Value } + +// Sort first by timestamp, then value, then labels. +func Compare(a, b Exemplar) int { + if a.Ts < b.Ts { + return -1 + } else if a.Ts > b.Ts { + return 1 + } + if a.Value < b.Value { + return -1 + } else if a.Value > b.Value { + return 1 + } + return labels.Compare(a.Labels, b.Labels) +} diff --git a/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go b/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go index 41873278cb..1c2f3ebeca 100644 --- a/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go +++ b/vendor/github.com/prometheus/prometheus/model/histogram/float_histogram.go @@ -94,8 +94,8 @@ func (h *FloatHistogram) CopyToSchema(targetSchema int32) *FloatHistogram { Sum: h.Sum, } - c.PositiveSpans, c.PositiveBuckets = mergeToSchema(h.PositiveSpans, h.PositiveBuckets, h.Schema, targetSchema) - c.NegativeSpans, c.NegativeBuckets = mergeToSchema(h.NegativeSpans, h.NegativeBuckets, h.Schema, targetSchema) + c.PositiveSpans, c.PositiveBuckets = reduceResolution(h.PositiveSpans, h.PositiveBuckets, h.Schema, targetSchema, false, false) + c.NegativeSpans, c.NegativeBuckets = reduceResolution(h.NegativeSpans, h.NegativeBuckets, h.Schema, targetSchema, false, false) return &c } @@ -268,17 +268,23 @@ func (h *FloatHistogram) Add(other *FloatHistogram) *FloatHistogram { h.Count += other.Count h.Sum += other.Sum - otherPositiveSpans := other.PositiveSpans - otherPositiveBuckets := other.PositiveBuckets - otherNegativeSpans := other.NegativeSpans - otherNegativeBuckets := other.NegativeBuckets - if other.Schema != h.Schema { - otherPositiveSpans, otherPositiveBuckets = mergeToSchema(other.PositiveSpans, other.PositiveBuckets, other.Schema, h.Schema) - otherNegativeSpans, otherNegativeBuckets = mergeToSchema(other.NegativeSpans, other.NegativeBuckets, other.Schema, h.Schema) + var ( + otherPositiveSpans = other.PositiveSpans + otherPositiveBuckets = other.PositiveBuckets + otherNegativeSpans = other.NegativeSpans + otherNegativeBuckets = other.NegativeBuckets + ) + + if other.Schema < h.Schema { + panic(fmt.Errorf("cannot add histogram with schema %d to %d", other.Schema, h.Schema)) + } else if other.Schema > h.Schema { + otherPositiveSpans, otherPositiveBuckets = reduceResolution(otherPositiveSpans, otherPositiveBuckets, other.Schema, h.Schema, false, false) + otherNegativeSpans, otherNegativeBuckets = reduceResolution(otherNegativeSpans, otherNegativeBuckets, other.Schema, h.Schema, false, false) } h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, h.PositiveSpans, h.PositiveBuckets, otherPositiveSpans, otherPositiveBuckets) h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, h.NegativeSpans, h.NegativeBuckets, otherNegativeSpans, otherNegativeBuckets) + return h } @@ -289,17 +295,23 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram { h.Count -= other.Count h.Sum -= other.Sum - otherPositiveSpans := other.PositiveSpans - otherPositiveBuckets := other.PositiveBuckets - otherNegativeSpans := other.NegativeSpans - otherNegativeBuckets := other.NegativeBuckets - if other.Schema != h.Schema { - otherPositiveSpans, otherPositiveBuckets = mergeToSchema(other.PositiveSpans, other.PositiveBuckets, other.Schema, h.Schema) - otherNegativeSpans, otherNegativeBuckets = mergeToSchema(other.NegativeSpans, other.NegativeBuckets, other.Schema, h.Schema) + var ( + otherPositiveSpans = other.PositiveSpans + otherPositiveBuckets = other.PositiveBuckets + otherNegativeSpans = other.NegativeSpans + otherNegativeBuckets = other.NegativeBuckets + ) + + if other.Schema < h.Schema { + panic(fmt.Errorf("cannot subtract histigram with schema %d to %d", other.Schema, h.Schema)) + } else if other.Schema > h.Schema { + otherPositiveSpans, otherPositiveBuckets = reduceResolution(otherPositiveSpans, otherPositiveBuckets, other.Schema, h.Schema, false, false) + otherNegativeSpans, otherNegativeBuckets = reduceResolution(otherNegativeSpans, otherNegativeBuckets, other.Schema, h.Schema, false, false) } h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, h.PositiveSpans, h.PositiveBuckets, otherPositiveSpans, otherPositiveBuckets) h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, h.NegativeSpans, h.NegativeBuckets, otherNegativeSpans, otherNegativeBuckets) + return h } @@ -307,13 +319,17 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) *FloatHistogram { // Exact match is when there are no new buckets (even empty) and no missing buckets, // and all the bucket values match. Spans can have different empty length spans in between, // but they must represent the same bucket layout to match. +// Sum, Count, ZeroCount and bucket values are compared based on their bit patterns +// because this method is about data equality rather than mathematical equality. func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool { if h2 == nil { return false } if h.Schema != h2.Schema || h.ZeroThreshold != h2.ZeroThreshold || - h.ZeroCount != h2.ZeroCount || h.Count != h2.Count || h.Sum != h2.Sum { + math.Float64bits(h.ZeroCount) != math.Float64bits(h2.ZeroCount) || + math.Float64bits(h.Count) != math.Float64bits(h2.Count) || + math.Float64bits(h.Sum) != math.Float64bits(h2.Sum) { return false } @@ -324,16 +340,44 @@ func (h *FloatHistogram) Equals(h2 *FloatHistogram) bool { return false } - if !bucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) { + if !floatBucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) { return false } - if !bucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) { + if !floatBucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) { return false } return true } +// Size returns the total size of the FloatHistogram, which includes the size of the pointer +// to FloatHistogram, all its fields, and all elements contained in slices. +// NOTE: this is only valid for 64 bit architectures. +func (h *FloatHistogram) Size() int { + // Size of each slice separately. + posSpanSize := len(h.PositiveSpans) * 8 // 8 bytes (int32 + uint32). + negSpanSize := len(h.NegativeSpans) * 8 // 8 bytes (int32 + uint32). + posBucketSize := len(h.PositiveBuckets) * 8 // 8 bytes (float64). + negBucketSize := len(h.NegativeBuckets) * 8 // 8 bytes (float64). + + // Total size of the struct. + + // fh is 8 bytes. + // fh.CounterResetHint is 4 bytes (1 byte bool + 3 bytes padding). + // fh.Schema is 4 bytes. + // fh.ZeroThreshold is 8 bytes. + // fh.ZeroCount is 8 bytes. + // fh.Count is 8 bytes. + // fh.Sum is 8 bytes. + // fh.PositiveSpans is 24 bytes. + // fh.NegativeSpans is 24 bytes. + // fh.PositiveBuckets is 24 bytes. + // fh.NegativeBuckets is 24 bytes. + structSize := 144 + + return structSize + posSpanSize + negSpanSize + posBucketSize + negBucketSize +} + // Compact eliminates empty buckets at the beginning and end of each span, then // merges spans that are consecutive or at most maxEmptyBuckets apart, and // finally splits spans that contain more consecutive empty buckets than @@ -434,25 +478,25 @@ func (h *FloatHistogram) DetectReset(previous *FloatHistogram) bool { } currIt := h.floatBucketIterator(true, h.ZeroThreshold, h.Schema) prevIt := previous.floatBucketIterator(true, h.ZeroThreshold, h.Schema) - if detectReset(currIt, prevIt) { + if detectReset(&currIt, &prevIt) { return true } currIt = h.floatBucketIterator(false, h.ZeroThreshold, h.Schema) prevIt = previous.floatBucketIterator(false, h.ZeroThreshold, h.Schema) - return detectReset(currIt, prevIt) + return detectReset(&currIt, &prevIt) } -func detectReset(currIt, prevIt BucketIterator[float64]) bool { +func detectReset(currIt, prevIt *floatBucketIterator) bool { if !prevIt.Next() { return false // If no buckets in previous histogram, nothing can be reset. } - prevBucket := prevIt.At() + prevBucket := prevIt.strippedAt() if !currIt.Next() { // No bucket in current, but at least one in previous // histogram. Check if any of those are non-zero, in which case // this is a reset. for { - if prevBucket.Count != 0 { + if prevBucket.count != 0 { return true } if !prevIt.Next() { @@ -460,10 +504,10 @@ func detectReset(currIt, prevIt BucketIterator[float64]) bool { } } } - currBucket := currIt.At() + currBucket := currIt.strippedAt() for { // Forward currIt until we find the bucket corresponding to prevBucket. - for currBucket.Index < prevBucket.Index { + for currBucket.index < prevBucket.index { if !currIt.Next() { // Reached end of currIt early, therefore // previous histogram has a bucket that the @@ -471,7 +515,7 @@ func detectReset(currIt, prevIt BucketIterator[float64]) bool { // remaining buckets in the previous histogram // are unpopulated, this is a reset. for { - if prevBucket.Count != 0 { + if prevBucket.count != 0 { return true } if !prevIt.Next() { @@ -479,18 +523,18 @@ func detectReset(currIt, prevIt BucketIterator[float64]) bool { } } } - currBucket = currIt.At() + currBucket = currIt.strippedAt() } - if currBucket.Index > prevBucket.Index { + if currBucket.index > prevBucket.index { // Previous histogram has a bucket the current one does // not have. If it's populated, it's a reset. - if prevBucket.Count != 0 { + if prevBucket.count != 0 { return true } } else { // We have reached corresponding buckets in both iterators. // We can finally compare the counts. - if currBucket.Count < prevBucket.Count { + if currBucket.count < prevBucket.count { return true } } @@ -498,35 +542,39 @@ func detectReset(currIt, prevIt BucketIterator[float64]) bool { // Reached end of prevIt without finding offending buckets. return false } - prevBucket = prevIt.At() + prevBucket = prevIt.strippedAt() } } // PositiveBucketIterator returns a BucketIterator to iterate over all positive // buckets in ascending order (starting next to the zero bucket and going up). func (h *FloatHistogram) PositiveBucketIterator() BucketIterator[float64] { - return h.floatBucketIterator(true, 0, h.Schema) + it := h.floatBucketIterator(true, 0, h.Schema) + return &it } // NegativeBucketIterator returns a BucketIterator to iterate over all negative // buckets in descending order (starting next to the zero bucket and going // down). func (h *FloatHistogram) NegativeBucketIterator() BucketIterator[float64] { - return h.floatBucketIterator(false, 0, h.Schema) + it := h.floatBucketIterator(false, 0, h.Schema) + return &it } // PositiveReverseBucketIterator returns a BucketIterator to iterate over all // positive buckets in descending order (starting at the highest bucket and // going down towards the zero bucket). func (h *FloatHistogram) PositiveReverseBucketIterator() BucketIterator[float64] { - return newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true) + it := newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true) + return &it } // NegativeReverseBucketIterator returns a BucketIterator to iterate over all // negative buckets in ascending order (starting at the lowest bucket and going // up towards the zero bucket). func (h *FloatHistogram) NegativeReverseBucketIterator() BucketIterator[float64] { - return newReverseFloatBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false) + it := newReverseFloatBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false) + return &it } // AllBucketIterator returns a BucketIterator to iterate over all negative, @@ -537,8 +585,8 @@ func (h *FloatHistogram) NegativeReverseBucketIterator() BucketIterator[float64] func (h *FloatHistogram) AllBucketIterator() BucketIterator[float64] { return &allFloatBucketIterator{ h: h, - leftIter: h.NegativeReverseBucketIterator(), - rightIter: h.PositiveBucketIterator(), + leftIter: newReverseFloatBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false), + rightIter: h.floatBucketIterator(true, 0, h.Schema), state: -1, } } @@ -551,12 +599,37 @@ func (h *FloatHistogram) AllBucketIterator() BucketIterator[float64] { func (h *FloatHistogram) AllReverseBucketIterator() BucketIterator[float64] { return &allFloatBucketIterator{ h: h, - leftIter: h.PositiveReverseBucketIterator(), - rightIter: h.NegativeBucketIterator(), + leftIter: newReverseFloatBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true), + rightIter: h.floatBucketIterator(false, 0, h.Schema), state: -1, } } +// Validate validates consistency between span and bucket slices. Also, buckets are checked +// against negative values. +// We do not check for h.Count being at least as large as the sum of the +// counts in the buckets because floating point precision issues can +// create false positives here. +func (h *FloatHistogram) Validate() error { + if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil { + return fmt.Errorf("negative side: %w", err) + } + if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil { + return fmt.Errorf("positive side: %w", err) + } + var nCount, pCount float64 + err := checkHistogramBuckets(h.NegativeBuckets, &nCount, false) + if err != nil { + return fmt.Errorf("negative side: %w", err) + } + err = checkHistogramBuckets(h.PositiveBuckets, &pCount, false) + if err != nil { + return fmt.Errorf("positive side: %w", err) + } + + return nil +} + // zeroCountForLargerThreshold returns what the histogram's zero count would be // if the ZeroThreshold had the provided larger (or equal) value. If the // provided value is less than the histogram's ZeroThreshold, the method panics. @@ -683,11 +756,11 @@ func (h *FloatHistogram) reconcileZeroBuckets(other *FloatHistogram) float64 { // targetSchema prior to iterating (without mutating FloatHistogram). func (h *FloatHistogram) floatBucketIterator( positive bool, absoluteStartValue float64, targetSchema int32, -) *floatBucketIterator { +) floatBucketIterator { if targetSchema > h.Schema { panic(fmt.Errorf("cannot merge from schema %d to %d", h.Schema, targetSchema)) } - i := &floatBucketIterator{ + i := floatBucketIterator{ baseBucketIterator: baseBucketIterator[float64, float64]{ schema: h.Schema, positive: positive, @@ -705,11 +778,11 @@ func (h *FloatHistogram) floatBucketIterator( return i } -// reverseFloatbucketiterator is a low-level constructor for reverse bucket iterators. +// reverseFloatBucketIterator is a low-level constructor for reverse bucket iterators. func newReverseFloatBucketIterator( spans []Span, buckets []float64, schema int32, positive bool, -) *reverseFloatBucketIterator { - r := &reverseFloatBucketIterator{ +) reverseFloatBucketIterator { + r := reverseFloatBucketIterator{ baseBucketIterator: baseBucketIterator[float64, float64]{ schema: schema, spans: spans, @@ -737,6 +810,8 @@ type floatBucketIterator struct { targetSchema int32 // targetSchema is the schema to merge to and must be ≤ schema. origIdx int32 // The bucket index within the original schema. absoluteStartValue float64 // Never return buckets with an upper bound ≤ this value. + + boundReachedStartValue bool // Has getBound reached absoluteStartValue already? } func (i *floatBucketIterator) At() Bucket[float64] { @@ -800,9 +875,10 @@ mergeLoop: // Merge together all buckets from the original schema that fall into } // Skip buckets before absoluteStartValue. // TODO(beorn7): Maybe do something more efficient than this recursive call. - if getBound(i.currIdx, i.targetSchema) <= i.absoluteStartValue { + if !i.boundReachedStartValue && getBound(i.currIdx, i.targetSchema) <= i.absoluteStartValue { return i.Next() } + i.boundReachedStartValue = true return true } @@ -843,8 +919,9 @@ func (i *reverseFloatBucketIterator) Next() bool { } type allFloatBucketIterator struct { - h *FloatHistogram - leftIter, rightIter BucketIterator[float64] + h *FloatHistogram + leftIter reverseFloatBucketIterator + rightIter floatBucketIterator // -1 means we are iterating negative buckets. // 0 means it is time for the zero bucket. // 1 means we are iterating positive buckets. @@ -910,69 +987,6 @@ func targetIdx(idx, originSchema, targetSchema int32) int32 { return ((idx - 1) >> (originSchema - targetSchema)) + 1 } -// mergeToSchema is used to merge a FloatHistogram's Spans and Buckets (no matter if -// positive or negative) from the original schema to the target schema. -// The target schema must be smaller than the original schema. -func mergeToSchema(originSpans []Span, originBuckets []float64, originSchema, targetSchema int32) ([]Span, []float64) { - var ( - targetSpans []Span // The spans in the target schema. - targetBuckets []float64 // The buckets in the target schema. - bucketIdx int32 // The index of bucket in the origin schema. - lastTargetBucketIdx int32 // The index of the last added target bucket. - origBucketIdx int // The position of a bucket in originBuckets slice. - ) - - for _, span := range originSpans { - // Determine the index of the first bucket in this span. - bucketIdx += span.Offset - for j := 0; j < int(span.Length); j++ { - // Determine the index of the bucket in the target schema from the index in the original schema. - targetBucketIdx := targetIdx(bucketIdx, originSchema, targetSchema) - - switch { - case len(targetSpans) == 0: - // This is the first span in the targetSpans. - span := Span{ - Offset: targetBucketIdx, - Length: 1, - } - targetSpans = append(targetSpans, span) - targetBuckets = append(targetBuckets, originBuckets[0]) - lastTargetBucketIdx = targetBucketIdx - - case lastTargetBucketIdx == targetBucketIdx: - // The current bucket has to be merged into the same target bucket as the previous bucket. - targetBuckets[len(targetBuckets)-1] += originBuckets[origBucketIdx] - - case (lastTargetBucketIdx + 1) == targetBucketIdx: - // The current bucket has to go into a new target bucket, - // and that bucket is next to the previous target bucket, - // so we add it to the current target span. - targetSpans[len(targetSpans)-1].Length++ - targetBuckets = append(targetBuckets, originBuckets[origBucketIdx]) - lastTargetBucketIdx++ - - case (lastTargetBucketIdx + 1) < targetBucketIdx: - // The current bucket has to go into a new target bucket, - // and that bucket is separated by a gap from the previous target bucket, - // so we need to add a new target span. - span := Span{ - Offset: targetBucketIdx - lastTargetBucketIdx - 1, - Length: 1, - } - targetSpans = append(targetSpans, span) - targetBuckets = append(targetBuckets, originBuckets[origBucketIdx]) - lastTargetBucketIdx = targetBucketIdx - } - - bucketIdx++ - origBucketIdx++ - } - } - - return targetSpans, targetBuckets -} - // addBuckets adds the buckets described by spansB/bucketsB to the buckets described by spansA/bucketsA, // creating missing buckets in spansA/bucketsA as needed. // It returns the resulting spans/buckets (which must be used instead of the original spansA/bucketsA, @@ -986,8 +1000,8 @@ func addBuckets( spansB []Span, bucketsB []float64, ) ([]Span, []float64) { var ( - iSpan int = -1 - iBucket int = -1 + iSpan = -1 + iBucket = -1 iInSpan int32 indexA int32 indexB int32 @@ -1020,17 +1034,16 @@ func addBuckets( spansA[0].Length++ spansA[0].Offset-- goto nextLoop - } else { - spansA = append(spansA, Span{}) - copy(spansA[1:], spansA) - spansA[0] = Span{Offset: indexB, Length: 1} - if len(spansA) > 1 { - // Convert the absolute offset in the formerly - // first span to a relative offset. - spansA[1].Offset -= indexB + 1 - } - goto nextLoop } + spansA = append(spansA, Span{}) + copy(spansA[1:], spansA) + spansA[0] = Span{Offset: indexB, Length: 1} + if len(spansA) > 1 { + // Convert the absolute offset in the formerly + // first span to a relative offset. + spansA[1].Offset -= indexB + 1 + } + goto nextLoop } else if spansA[0].Offset == indexB { // Just add to first bucket. bucketsA[0] += bucketB @@ -1048,48 +1061,47 @@ func addBuckets( iInSpan += deltaIndex bucketsA[iBucket] += bucketB break - } else { - deltaIndex -= remainingInSpan - iBucket += int(remainingInSpan) - iSpan++ - if iSpan == len(spansA) || deltaIndex < spansA[iSpan].Offset { - // Bucket is in gap behind previous span (or there are no further spans). - bucketsA = append(bucketsA, 0) - copy(bucketsA[iBucket+1:], bucketsA[iBucket:]) - bucketsA[iBucket] = bucketB - switch { - case deltaIndex == 0: - // Directly after previous span, extend previous span. - if iSpan < len(spansA) { - spansA[iSpan].Offset-- - } - iSpan-- - iInSpan = int32(spansA[iSpan].Length) - spansA[iSpan].Length++ - goto nextLoop - case iSpan < len(spansA) && deltaIndex == spansA[iSpan].Offset-1: - // Directly before next span, extend next span. - iInSpan = 0 + } + deltaIndex -= remainingInSpan + iBucket += int(remainingInSpan) + iSpan++ + if iSpan == len(spansA) || deltaIndex < spansA[iSpan].Offset { + // Bucket is in gap behind previous span (or there are no further spans). + bucketsA = append(bucketsA, 0) + copy(bucketsA[iBucket+1:], bucketsA[iBucket:]) + bucketsA[iBucket] = bucketB + switch { + case deltaIndex == 0: + // Directly after previous span, extend previous span. + if iSpan < len(spansA) { spansA[iSpan].Offset-- - spansA[iSpan].Length++ - goto nextLoop - default: - // No next span, or next span is not directly adjacent to new bucket. - // Add new span. - iInSpan = 0 - if iSpan < len(spansA) { - spansA[iSpan].Offset -= deltaIndex + 1 - } - spansA = append(spansA, Span{}) - copy(spansA[iSpan+1:], spansA[iSpan:]) - spansA[iSpan] = Span{Length: 1, Offset: deltaIndex} - goto nextLoop } - } else { - // Try start of next span. - deltaIndex -= spansA[iSpan].Offset + iSpan-- + iInSpan = int32(spansA[iSpan].Length) + spansA[iSpan].Length++ + goto nextLoop + case iSpan < len(spansA) && deltaIndex == spansA[iSpan].Offset-1: + // Directly before next span, extend next span. + iInSpan = 0 + spansA[iSpan].Offset-- + spansA[iSpan].Length++ + goto nextLoop + default: + // No next span, or next span is not directly adjacent to new bucket. + // Add new span. iInSpan = 0 + if iSpan < len(spansA) { + spansA[iSpan].Offset -= deltaIndex + 1 + } + spansA = append(spansA, Span{}) + copy(spansA[iSpan+1:], spansA[iSpan:]) + spansA[iSpan] = Span{Length: 1, Offset: deltaIndex} + goto nextLoop } + } else { + // Try start of next span. + deltaIndex -= spansA[iSpan].Offset + iInSpan = 0 } } @@ -1102,3 +1114,29 @@ func addBuckets( return spansA, bucketsA } + +func floatBucketsMatch(b1, b2 []float64) bool { + if len(b1) != len(b2) { + return false + } + for i, b := range b1 { + if math.Float64bits(b) != math.Float64bits(b2[i]) { + return false + } + } + return true +} + +// ReduceResolution reduces the float histogram's spans, buckets into target schema. +// The target schema must be smaller than the current float histogram's schema. +func (h *FloatHistogram) ReduceResolution(targetSchema int32) *FloatHistogram { + if targetSchema >= h.Schema { + panic(fmt.Errorf("cannot reduce resolution from schema %d to %d", h.Schema, targetSchema)) + } + + h.PositiveSpans, h.PositiveBuckets = reduceResolution(h.PositiveSpans, h.PositiveBuckets, h.Schema, targetSchema, false, true) + h.NegativeSpans, h.NegativeBuckets = reduceResolution(h.NegativeSpans, h.NegativeBuckets, h.Schema, targetSchema, false, true) + + h.Schema = targetSchema + return h +} diff --git a/vendor/github.com/prometheus/prometheus/model/histogram/generic.go b/vendor/github.com/prometheus/prometheus/model/histogram/generic.go index dad54cb069..7e1cc4b605 100644 --- a/vendor/github.com/prometheus/prometheus/model/histogram/generic.go +++ b/vendor/github.com/prometheus/prometheus/model/histogram/generic.go @@ -14,11 +14,20 @@ package histogram import ( + "errors" "fmt" "math" "strings" ) +var ( + ErrHistogramCountNotBigEnough = errors.New("histogram's observation count should be at least the number of observations found in the buckets") + ErrHistogramCountMismatch = errors.New("histogram's observation count should equal the number of observations found in the buckets (in absence of NaN)") + ErrHistogramNegativeBucketCount = errors.New("histogram has a bucket whose observation count is negative") + ErrHistogramSpanNegativeOffset = errors.New("histogram has a span whose offset is negative") + ErrHistogramSpansBucketsMismatch = errors.New("histogram spans specify different number of buckets than provided") +) + // BucketCount is a type constraint for the count in a bucket, which can be // float64 (for type FloatHistogram) or uint64 (for type Histogram). type BucketCount interface { @@ -31,7 +40,7 @@ type BucketCount interface { // absolute counts directly). Go type parameters don't allow type // specialization. Therefore, where special treatment of deltas between buckets // vs. absolute counts is important, this information has to be provided as a -// separate boolean parameter "deltaBuckets" +// separate boolean parameter "deltaBuckets". type InternalBucketCount interface { float64 | int64 } @@ -53,6 +62,13 @@ type Bucket[BC BucketCount] struct { Index int32 } +// strippedBucket is Bucket without bound values (which are expensive to calculate +// and not used in certain use cases). +type strippedBucket[BC BucketCount] struct { + count BC + index int32 +} + // String returns a string representation of a Bucket, using the usual // mathematical notation of '['/']' for inclusive bounds and '('/')' for // non-inclusive bounds. @@ -101,13 +117,12 @@ type baseBucketIterator[BC BucketCount, IBC InternalBucketCount] struct { currIdx int32 // The actual bucket index. } -func (b baseBucketIterator[BC, IBC]) At() Bucket[BC] { +func (b *baseBucketIterator[BC, IBC]) At() Bucket[BC] { return b.at(b.schema) } -// at is an internal version of the exported At to enable using a different -// schema. -func (b baseBucketIterator[BC, IBC]) at(schema int32) Bucket[BC] { +// at is an internal version of the exported At to enable using a different schema. +func (b *baseBucketIterator[BC, IBC]) at(schema int32) Bucket[BC] { bucket := Bucket[BC]{ Count: BC(b.currCount), Index: b.currIdx, @@ -124,6 +139,14 @@ func (b baseBucketIterator[BC, IBC]) at(schema int32) Bucket[BC] { return bucket } +// strippedAt returns current strippedBucket (which lacks bucket bounds but is cheaper to compute). +func (b *baseBucketIterator[BC, IBC]) strippedAt() strippedBucket[BC] { + return strippedBucket[BC]{ + count: BC(b.currCount), + index: b.currIdx, + } +} + // compactBuckets is a generic function used by both Histogram.Compact and // FloatHistogram.Compact. Set deltaBuckets to true if the provided buckets are // deltas. Set it to false if the buckets contain absolute counts. @@ -333,16 +356,41 @@ func compactBuckets[IBC InternalBucketCount](buckets []IBC, spans []Span, maxEmp return buckets, spans } -func bucketsMatch[IBC InternalBucketCount](b1, b2 []IBC) bool { - if len(b1) != len(b2) { - return false +func checkHistogramSpans(spans []Span, numBuckets int) error { + var spanBuckets int + for n, span := range spans { + if n > 0 && span.Offset < 0 { + return fmt.Errorf("span number %d with offset %d: %w", n+1, span.Offset, ErrHistogramSpanNegativeOffset) + } + spanBuckets += int(span.Length) + } + if spanBuckets != numBuckets { + return fmt.Errorf("spans need %d buckets, have %d buckets: %w", spanBuckets, numBuckets, ErrHistogramSpansBucketsMismatch) + } + return nil +} + +func checkHistogramBuckets[BC BucketCount, IBC InternalBucketCount](buckets []IBC, count *BC, deltas bool) error { + if len(buckets) == 0 { + return nil } - for i, b := range b1 { - if b != b2[i] { - return false + + var last IBC + for i := 0; i < len(buckets); i++ { + var c IBC + if deltas { + c = last + buckets[i] + } else { + c = buckets[i] + } + if c < 0 { + return fmt.Errorf("bucket number %d has observation count of %v: %w", i+1, c, ErrHistogramNegativeBucketCount) } + last = c + *count += BC(c) } - return true + + return nil } func getBound(idx, schema int32) float64 { @@ -552,3 +600,106 @@ var exponentialBounds = [][]float64{ 0.9892280131939752, 0.9919100824251095, 0.9945994234836328, 0.9972960560854698, }, } + +// reduceResolution reduces the input spans, buckets in origin schema to the spans, buckets in target schema. +// The target schema must be smaller than the original schema. +// Set deltaBuckets to true if the provided buckets are +// deltas. Set it to false if the buckets contain absolute counts. +// Set inplace to true to reuse input slices and avoid allocations (otherwise +// new slices will be allocated for result). +func reduceResolution[IBC InternalBucketCount]( + originSpans []Span, + originBuckets []IBC, + originSchema, + targetSchema int32, + deltaBuckets bool, + inplace bool, +) ([]Span, []IBC) { + var ( + targetSpans []Span // The spans in the target schema. + targetBuckets []IBC // The bucket counts in the target schema. + bucketIdx int32 // The index of bucket in the origin schema. + bucketCountIdx int // The position of a bucket in origin bucket count slice `originBuckets`. + targetBucketIdx int32 // The index of bucket in the target schema. + lastBucketCount IBC // The last visited bucket's count in the origin schema. + lastTargetBucketIdx int32 // The index of the last added target bucket. + lastTargetBucketCount IBC + ) + + if inplace { + // Slice reuse is safe because when reducing the resolution, + // target slices don't grow faster than origin slices are being read. + targetSpans = originSpans[:0] + targetBuckets = originBuckets[:0] + } + + for _, span := range originSpans { + // Determine the index of the first bucket in this span. + bucketIdx += span.Offset + for j := 0; j < int(span.Length); j++ { + // Determine the index of the bucket in the target schema from the index in the original schema. + targetBucketIdx = targetIdx(bucketIdx, originSchema, targetSchema) + + switch { + case len(targetSpans) == 0: + // This is the first span in the targetSpans. + span := Span{ + Offset: targetBucketIdx, + Length: 1, + } + targetSpans = append(targetSpans, span) + targetBuckets = append(targetBuckets, originBuckets[bucketCountIdx]) + lastTargetBucketIdx = targetBucketIdx + lastBucketCount = originBuckets[bucketCountIdx] + lastTargetBucketCount = originBuckets[bucketCountIdx] + + case lastTargetBucketIdx == targetBucketIdx: + // The current bucket has to be merged into the same target bucket as the previous bucket. + if deltaBuckets { + lastBucketCount += originBuckets[bucketCountIdx] + targetBuckets[len(targetBuckets)-1] += lastBucketCount + lastTargetBucketCount += lastBucketCount + } else { + targetBuckets[len(targetBuckets)-1] += originBuckets[bucketCountIdx] + } + + case (lastTargetBucketIdx + 1) == targetBucketIdx: + // The current bucket has to go into a new target bucket, + // and that bucket is next to the previous target bucket, + // so we add it to the current target span. + targetSpans[len(targetSpans)-1].Length++ + lastTargetBucketIdx++ + if deltaBuckets { + lastBucketCount += originBuckets[bucketCountIdx] + targetBuckets = append(targetBuckets, lastBucketCount-lastTargetBucketCount) + lastTargetBucketCount = lastBucketCount + } else { + targetBuckets = append(targetBuckets, originBuckets[bucketCountIdx]) + } + + case (lastTargetBucketIdx + 1) < targetBucketIdx: + // The current bucket has to go into a new target bucket, + // and that bucket is separated by a gap from the previous target bucket, + // so we need to add a new target span. + span := Span{ + Offset: targetBucketIdx - lastTargetBucketIdx - 1, + Length: 1, + } + targetSpans = append(targetSpans, span) + lastTargetBucketIdx = targetBucketIdx + if deltaBuckets { + lastBucketCount += originBuckets[bucketCountIdx] + targetBuckets = append(targetBuckets, lastBucketCount-lastTargetBucketCount) + lastTargetBucketCount = lastBucketCount + } else { + targetBuckets = append(targetBuckets, originBuckets[bucketCountIdx]) + } + } + + bucketIdx++ + bucketCountIdx++ + } + } + + return targetSpans, targetBuckets +} diff --git a/vendor/github.com/prometheus/prometheus/model/histogram/histogram.go b/vendor/github.com/prometheus/prometheus/model/histogram/histogram.go index 6d425307c5..fb0185a638 100644 --- a/vendor/github.com/prometheus/prometheus/model/histogram/histogram.go +++ b/vendor/github.com/prometheus/prometheus/model/histogram/histogram.go @@ -17,6 +17,8 @@ import ( "fmt" "math" "strings" + + "golang.org/x/exp/slices" ) // CounterResetHint contains the known information about a counter reset, @@ -148,13 +150,15 @@ func (h *Histogram) ZeroBucket() Bucket[uint64] { // PositiveBucketIterator returns a BucketIterator to iterate over all positive // buckets in ascending order (starting next to the zero bucket and going up). func (h *Histogram) PositiveBucketIterator() BucketIterator[uint64] { - return newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true) + it := newRegularBucketIterator(h.PositiveSpans, h.PositiveBuckets, h.Schema, true) + return &it } // NegativeBucketIterator returns a BucketIterator to iterate over all negative // buckets in descending order (starting next to the zero bucket and going down). func (h *Histogram) NegativeBucketIterator() BucketIterator[uint64] { - return newRegularBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false) + it := newRegularBucketIterator(h.NegativeSpans, h.NegativeBuckets, h.Schema, false) + return &it } // CumulativeBucketIterator returns a BucketIterator to iterate over a @@ -172,13 +176,16 @@ func (h *Histogram) CumulativeBucketIterator() BucketIterator[uint64] { // Exact match is when there are no new buckets (even empty) and no missing buckets, // and all the bucket values match. Spans can have different empty length spans in between, // but they must represent the same bucket layout to match. +// Sum is compared based on its bit pattern because this method +// is about data equality rather than mathematical equality. func (h *Histogram) Equals(h2 *Histogram) bool { if h2 == nil { return false } if h.Schema != h2.Schema || h.ZeroThreshold != h2.ZeroThreshold || - h.ZeroCount != h2.ZeroCount || h.Count != h2.Count || h.Sum != h2.Sum { + h.ZeroCount != h2.ZeroCount || h.Count != h2.Count || + math.Float64bits(h.Sum) != math.Float64bits(h2.Sum) { return false } @@ -189,10 +196,10 @@ func (h *Histogram) Equals(h2 *Histogram) bool { return false } - if !bucketsMatch(h.PositiveBuckets, h2.PositiveBuckets) { + if !slices.Equal(h.PositiveBuckets, h2.PositiveBuckets) { return false } - if !bucketsMatch(h.NegativeBuckets, h2.NegativeBuckets) { + if !slices.Equal(h.NegativeBuckets, h2.NegativeBuckets) { return false } @@ -321,18 +328,56 @@ func (h *Histogram) ToFloat() *FloatHistogram { } } +// Validate validates consistency between span and bucket slices. Also, buckets are checked +// against negative values. +// For histograms that have not observed any NaN values (based on IsNaN(h.Sum) check), a +// strict h.Count = nCount + pCount + h.ZeroCount check is performed. +// Otherwise, only a lower bound check will be done (h.Count >= nCount + pCount + h.ZeroCount), +// because NaN observations do not increment the values of buckets (but they do increment +// the total h.Count). +func (h *Histogram) Validate() error { + if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil { + return fmt.Errorf("negative side: %w", err) + } + if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil { + return fmt.Errorf("positive side: %w", err) + } + var nCount, pCount uint64 + err := checkHistogramBuckets(h.NegativeBuckets, &nCount, true) + if err != nil { + return fmt.Errorf("negative side: %w", err) + } + err = checkHistogramBuckets(h.PositiveBuckets, &pCount, true) + if err != nil { + return fmt.Errorf("positive side: %w", err) + } + + sumOfBuckets := nCount + pCount + h.ZeroCount + if math.IsNaN(h.Sum) { + if sumOfBuckets > h.Count { + return fmt.Errorf("%d observations found in buckets, but the Count field is %d: %w", sumOfBuckets, h.Count, ErrHistogramCountNotBigEnough) + } + } else { + if sumOfBuckets != h.Count { + return fmt.Errorf("%d observations found in buckets, but the Count field is %d: %w", sumOfBuckets, h.Count, ErrHistogramCountMismatch) + } + } + + return nil +} + type regularBucketIterator struct { baseBucketIterator[uint64, int64] } -func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool) *regularBucketIterator { +func newRegularBucketIterator(spans []Span, buckets []int64, schema int32, positive bool) regularBucketIterator { i := baseBucketIterator[uint64, int64]{ schema: schema, spans: spans, buckets: buckets, positive: positive, } - return ®ularBucketIterator{i} + return regularBucketIterator{i} } func (r *regularBucketIterator) Next() bool { @@ -448,3 +493,20 @@ func (c *cumulativeBucketIterator) At() Bucket[uint64] { Index: c.currIdx - 1, } } + +// ReduceResolution reduces the histogram's spans, buckets into target schema. +// The target schema must be smaller than the current histogram's schema. +func (h *Histogram) ReduceResolution(targetSchema int32) *Histogram { + if targetSchema >= h.Schema { + panic(fmt.Errorf("cannot reduce resolution from schema %d to %d", h.Schema, targetSchema)) + } + + h.PositiveSpans, h.PositiveBuckets = reduceResolution( + h.PositiveSpans, h.PositiveBuckets, h.Schema, targetSchema, true, true, + ) + h.NegativeSpans, h.NegativeBuckets = reduceResolution( + h.NegativeSpans, h.NegativeBuckets, h.Schema, targetSchema, true, true, + ) + h.Schema = targetSchema + return h +} diff --git a/vendor/github.com/prometheus/prometheus/model/histogram/test_utils.go b/vendor/github.com/prometheus/prometheus/model/histogram/test_utils.go new file mode 100644 index 0000000000..9e9a711c29 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/model/histogram/test_utils.go @@ -0,0 +1,52 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package histogram + +// GenerateBigTestHistograms generates a slice of histograms with given number of buckets each. +func GenerateBigTestHistograms(numHistograms, numBuckets int) []*Histogram { + numSpans := numBuckets / 10 + bucketsPerSide := numBuckets / 2 + spanLength := uint32(bucketsPerSide / numSpans) + // Given all bucket deltas are 1, sum bucketsPerSide + 1. + observationCount := bucketsPerSide * (1 + bucketsPerSide) + + var histograms []*Histogram + for i := 0; i < numHistograms; i++ { + h := &Histogram{ + Count: uint64(i + observationCount), + ZeroCount: uint64(i), + ZeroThreshold: 1e-128, + Sum: 18.4 * float64(i+1), + Schema: 2, + NegativeSpans: make([]Span, numSpans), + PositiveSpans: make([]Span, numSpans), + NegativeBuckets: make([]int64, bucketsPerSide), + PositiveBuckets: make([]int64, bucketsPerSide), + } + + for j := 0; j < numSpans; j++ { + s := Span{Offset: 1, Length: spanLength} + h.NegativeSpans[j] = s + h.PositiveSpans[j] = s + } + + for j := 0; j < bucketsPerSide; j++ { + h.NegativeBuckets[j] = 1 + h.PositiveBuckets[j] = 1 + } + + histograms = append(histograms, h) + } + return histograms +} diff --git a/vendor/github.com/prometheus/prometheus/model/labels/labels.go b/vendor/github.com/prometheus/prometheus/model/labels/labels.go index 3dc3049b1c..bf67224bba 100644 --- a/vendor/github.com/prometheus/prometheus/model/labels/labels.go +++ b/vendor/github.com/prometheus/prometheus/model/labels/labels.go @@ -17,32 +17,12 @@ package labels import ( "bytes" - "encoding/json" - "strconv" "strings" "github.com/cespare/xxhash/v2" - "github.com/prometheus/common/model" "golang.org/x/exp/slices" ) -// Well-known label names used by Prometheus components. -const ( - MetricName = "__name__" - AlertName = "alertname" - BucketLabel = "le" - InstanceName = "instance" - - labelSep = '\xfe' -) - -var seps = []byte{'\xff'} - -// Label is a key/value pair of strings. -type Label struct { - Name, Value string -} - // Labels is a sorted set of labels. Order has to be guaranteed upon // instantiation. type Labels []Label @@ -51,23 +31,6 @@ func (ls Labels) Len() int { return len(ls) } func (ls Labels) Swap(i, j int) { ls[i], ls[j] = ls[j], ls[i] } func (ls Labels) Less(i, j int) bool { return ls[i].Name < ls[j].Name } -func (ls Labels) String() string { - var b bytes.Buffer - - b.WriteByte('{') - for i, l := range ls { - if i > 0 { - b.WriteByte(',') - b.WriteByte(' ') - } - b.WriteString(l.Name) - b.WriteByte('=') - b.WriteString(strconv.Quote(l.Value)) - } - b.WriteByte('}') - return b.String() -} - // Bytes returns ls as a byte slice. // It uses an byte invalid character as a separator and so should not be used for printing. func (ls Labels) Bytes(buf []byte) []byte { @@ -84,40 +47,6 @@ func (ls Labels) Bytes(buf []byte) []byte { return b.Bytes() } -// MarshalJSON implements json.Marshaler. -func (ls Labels) MarshalJSON() ([]byte, error) { - return json.Marshal(ls.Map()) -} - -// UnmarshalJSON implements json.Unmarshaler. -func (ls *Labels) UnmarshalJSON(b []byte) error { - var m map[string]string - - if err := json.Unmarshal(b, &m); err != nil { - return err - } - - *ls = FromMap(m) - return nil -} - -// MarshalYAML implements yaml.Marshaler. -func (ls Labels) MarshalYAML() (interface{}, error) { - return ls.Map(), nil -} - -// UnmarshalYAML implements yaml.Unmarshaler. -func (ls *Labels) UnmarshalYAML(unmarshal func(interface{}) error) error { - var m map[string]string - - if err := unmarshal(&m); err != nil { - return err - } - - *ls = FromMap(m) - return nil -} - // MatchLabels returns a subset of Labels that matches/does not match with the provided label names based on the 'on' boolean. // If on is set to true, it returns the subset of labels that match with the provided label names and its inverse when 'on' is set to false. func (ls Labels) MatchLabels(on bool, names ...string) Labels { @@ -318,19 +247,6 @@ func (ls Labels) WithoutEmpty() Labels { return ls } -// IsValid checks if the metric name or label names are valid. -func (ls Labels) IsValid() bool { - for _, l := range ls { - if l.Name == model.MetricNameLabel && !model.IsValidMetricName(model.LabelValue(l.Value)) { - return false - } - if !model.LabelName(l.Name).IsValid() || !model.LabelValue(l.Value).IsValid() { - return false - } - } - return true -} - // Equal returns whether the two label sets are equal. func Equal(ls, o Labels) bool { if len(ls) != len(o) { @@ -344,15 +260,6 @@ func Equal(ls, o Labels) bool { return true } -// Map returns a string map of the labels. -func (ls Labels) Map() map[string]string { - m := make(map[string]string, len(ls)) - for _, l := range ls { - m[l.Name] = l.Value - } - return m -} - // EmptyLabels returns n empty Labels value, for convenience. func EmptyLabels() Labels { return Labels{} @@ -368,15 +275,6 @@ func New(ls ...Label) Labels { return set } -// FromMap returns new sorted Labels from the given map. -func FromMap(m map[string]string) Labels { - l := make([]Label, 0, len(m)) - for k, v := range m { - l = append(l, Label{Name: k, Value: v}) - } - return New(l...) -} - // FromStrings creates new labels from pairs of strings. func FromStrings(ss ...string) Labels { if len(ss)%2 != 0 { @@ -460,118 +358,6 @@ func (ls Labels) ReleaseStrings(release func(string)) { } } -// Builder allows modifying Labels. -type Builder struct { - base Labels - del []string - add []Label -} - -// NewBuilder returns a new LabelsBuilder. -func NewBuilder(base Labels) *Builder { - b := &Builder{ - del: make([]string, 0, 5), - add: make([]Label, 0, 5), - } - b.Reset(base) - return b -} - -// Reset clears all current state for the builder. -func (b *Builder) Reset(base Labels) { - b.base = base - b.del = b.del[:0] - b.add = b.add[:0] - for _, l := range b.base { - if l.Value == "" { - b.del = append(b.del, l.Name) - } - } -} - -// Del deletes the label of the given name. -func (b *Builder) Del(ns ...string) *Builder { - for _, n := range ns { - for i, a := range b.add { - if a.Name == n { - b.add = append(b.add[:i], b.add[i+1:]...) - } - } - b.del = append(b.del, n) - } - return b -} - -// Keep removes all labels from the base except those with the given names. -func (b *Builder) Keep(ns ...string) *Builder { -Outer: - for _, l := range b.base { - for _, n := range ns { - if l.Name == n { - continue Outer - } - } - b.del = append(b.del, l.Name) - } - return b -} - -// Set the name/value pair as a label. A value of "" means delete that label. -func (b *Builder) Set(n, v string) *Builder { - if v == "" { - // Empty labels are the same as missing labels. - return b.Del(n) - } - for i, a := range b.add { - if a.Name == n { - b.add[i].Value = v - return b - } - } - b.add = append(b.add, Label{Name: n, Value: v}) - - return b -} - -func (b *Builder) Get(n string) string { - // Del() removes entries from .add but Set() does not remove from .del, so check .add first. - for _, a := range b.add { - if a.Name == n { - return a.Value - } - } - if slices.Contains(b.del, n) { - return "" - } - return b.base.Get(n) -} - -// Range calls f on each label in the Builder. -func (b *Builder) Range(f func(l Label)) { - // Stack-based arrays to avoid heap allocation in most cases. - var addStack [128]Label - var delStack [128]string - // Take a copy of add and del, so they are unaffected by calls to Set() or Del(). - origAdd, origDel := append(addStack[:0], b.add...), append(delStack[:0], b.del...) - b.base.Range(func(l Label) { - if !slices.Contains(origDel, l.Name) && !contains(origAdd, l.Name) { - f(l) - } - }) - for _, a := range origAdd { - f(a) - } -} - -func contains(s []Label, n string) bool { - for _, a := range s { - if a.Name == n { - return true - } - } - return false -} - // Labels returns the labels from the builder. // If no modifications were made, the original labels are returned. func (b *Builder) Labels() Labels { @@ -617,6 +403,13 @@ func (b *ScratchBuilder) Add(name, value string) { b.add = append(b.add, Label{Name: name, Value: value}) } +// Add a name/value pair, using []byte instead of string. +// The '-tags stringlabels' version of this function is unsafe, hence the name. +// This version is safe - it copies the strings immediately - but we keep the same name so everything compiles. +func (b *ScratchBuilder) UnsafeAddBytes(name, value []byte) { + b.add = append(b.add, Label{Name: string(name), Value: string(value)}) +} + // Sort the labels added so far by name. func (b *ScratchBuilder) Sort() { slices.SortFunc(b.add, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) diff --git a/vendor/github.com/prometheus/prometheus/model/labels/labels_common.go b/vendor/github.com/prometheus/prometheus/model/labels/labels_common.go new file mode 100644 index 0000000000..2a722b84cc --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/model/labels/labels_common.go @@ -0,0 +1,235 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labels + +import ( + "bytes" + "encoding/json" + "strconv" + + "github.com/prometheus/common/model" + "golang.org/x/exp/slices" +) + +const ( + MetricName = "__name__" + AlertName = "alertname" + BucketLabel = "le" + InstanceName = "instance" + + labelSep = '\xfe' +) + +var seps = []byte{'\xff'} + +// Label is a key/value pair of strings. +type Label struct { + Name, Value string +} + +func (ls Labels) String() string { + var b bytes.Buffer + + b.WriteByte('{') + i := 0 + ls.Range(func(l Label) { + if i > 0 { + b.WriteByte(',') + b.WriteByte(' ') + } + b.WriteString(l.Name) + b.WriteByte('=') + b.WriteString(strconv.Quote(l.Value)) + i++ + }) + b.WriteByte('}') + return b.String() +} + +// MarshalJSON implements json.Marshaler. +func (ls Labels) MarshalJSON() ([]byte, error) { + return json.Marshal(ls.Map()) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (ls *Labels) UnmarshalJSON(b []byte) error { + var m map[string]string + + if err := json.Unmarshal(b, &m); err != nil { + return err + } + + *ls = FromMap(m) + return nil +} + +// MarshalYAML implements yaml.Marshaler. +func (ls Labels) MarshalYAML() (interface{}, error) { + return ls.Map(), nil +} + +// UnmarshalYAML implements yaml.Unmarshaler. +func (ls *Labels) UnmarshalYAML(unmarshal func(interface{}) error) error { + var m map[string]string + + if err := unmarshal(&m); err != nil { + return err + } + + *ls = FromMap(m) + return nil +} + +// IsValid checks if the metric name or label names are valid. +func (ls Labels) IsValid() bool { + err := ls.Validate(func(l Label) error { + if l.Name == model.MetricNameLabel && !model.IsValidMetricName(model.LabelValue(l.Value)) { + return strconv.ErrSyntax + } + if !model.LabelName(l.Name).IsValid() || !model.LabelValue(l.Value).IsValid() { + return strconv.ErrSyntax + } + return nil + }) + return err == nil +} + +// Map returns a string map of the labels. +func (ls Labels) Map() map[string]string { + m := make(map[string]string) + ls.Range(func(l Label) { + m[l.Name] = l.Value + }) + return m +} + +// FromMap returns new sorted Labels from the given map. +func FromMap(m map[string]string) Labels { + l := make([]Label, 0, len(m)) + for k, v := range m { + l = append(l, Label{Name: k, Value: v}) + } + return New(l...) +} + +// Builder allows modifying Labels. +type Builder struct { + base Labels + del []string + add []Label +} + +// NewBuilder returns a new LabelsBuilder. +func NewBuilder(base Labels) *Builder { + b := &Builder{ + del: make([]string, 0, 5), + add: make([]Label, 0, 5), + } + b.Reset(base) + return b +} + +// Reset clears all current state for the builder. +func (b *Builder) Reset(base Labels) { + b.base = base + b.del = b.del[:0] + b.add = b.add[:0] + b.base.Range(func(l Label) { + if l.Value == "" { + b.del = append(b.del, l.Name) + } + }) +} + +// Del deletes the label of the given name. +func (b *Builder) Del(ns ...string) *Builder { + for _, n := range ns { + for i, a := range b.add { + if a.Name == n { + b.add = append(b.add[:i], b.add[i+1:]...) + } + } + b.del = append(b.del, n) + } + return b +} + +// Keep removes all labels from the base except those with the given names. +func (b *Builder) Keep(ns ...string) *Builder { + b.base.Range(func(l Label) { + for _, n := range ns { + if l.Name == n { + return + } + } + b.del = append(b.del, l.Name) + }) + return b +} + +// Set the name/value pair as a label. A value of "" means delete that label. +func (b *Builder) Set(n, v string) *Builder { + if v == "" { + // Empty labels are the same as missing labels. + return b.Del(n) + } + for i, a := range b.add { + if a.Name == n { + b.add[i].Value = v + return b + } + } + b.add = append(b.add, Label{Name: n, Value: v}) + + return b +} + +func (b *Builder) Get(n string) string { + // Del() removes entries from .add but Set() does not remove from .del, so check .add first. + for _, a := range b.add { + if a.Name == n { + return a.Value + } + } + if slices.Contains(b.del, n) { + return "" + } + return b.base.Get(n) +} + +// Range calls f on each label in the Builder. +func (b *Builder) Range(f func(l Label)) { + // Stack-based arrays to avoid heap allocation in most cases. + var addStack [128]Label + var delStack [128]string + // Take a copy of add and del, so they are unaffected by calls to Set() or Del(). + origAdd, origDel := append(addStack[:0], b.add...), append(delStack[:0], b.del...) + b.base.Range(func(l Label) { + if !slices.Contains(origDel, l.Name) && !contains(origAdd, l.Name) { + f(l) + } + }) + for _, a := range origAdd { + f(a) + } +} + +func contains(s []Label, n string) bool { + for _, a := range s { + if a.Name == n { + return true + } + } + return false +} diff --git a/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go b/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go index cc6bfcc700..d79a836796 100644 --- a/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go +++ b/vendor/github.com/prometheus/prometheus/model/labels/labels_stringlabels.go @@ -16,33 +16,14 @@ package labels import ( - "bytes" - "encoding/json" "reflect" - "strconv" "strings" "unsafe" "github.com/cespare/xxhash/v2" - "github.com/prometheus/common/model" "golang.org/x/exp/slices" ) -// Well-known label names used by Prometheus components. -const ( - MetricName = "__name__" - AlertName = "alertname" - BucketLabel = "le" - InstanceName = "instance" -) - -var seps = []byte{'\xff'} - -// Label is a key/value pair of strings. -type Label struct { - Name, Value string -} - // Labels is implemented by a single flat string holding name/value pairs. // Each name and value is preceded by its length in varint encoding. // Names are in order. @@ -77,26 +58,6 @@ func decodeString(data string, index int) (string, int) { return data[index : index+size], index + size } -func (ls Labels) String() string { - var b bytes.Buffer - - b.WriteByte('{') - for i := 0; i < len(ls.data); { - if i > 0 { - b.WriteByte(',') - b.WriteByte(' ') - } - var name, value string - name, i = decodeString(ls.data, i) - value, i = decodeString(ls.data, i) - b.WriteString(name) - b.WriteByte('=') - b.WriteString(strconv.Quote(value)) - } - b.WriteByte('}') - return b.String() -} - // Bytes returns ls as a byte slice. // It uses non-printing characters and so should not be used for printing. func (ls Labels) Bytes(buf []byte) []byte { @@ -109,45 +70,11 @@ func (ls Labels) Bytes(buf []byte) []byte { return buf } -// MarshalJSON implements json.Marshaler. -func (ls Labels) MarshalJSON() ([]byte, error) { - return json.Marshal(ls.Map()) -} - -// UnmarshalJSON implements json.Unmarshaler. -func (ls *Labels) UnmarshalJSON(b []byte) error { - var m map[string]string - - if err := json.Unmarshal(b, &m); err != nil { - return err - } - - *ls = FromMap(m) - return nil -} - -// MarshalYAML implements yaml.Marshaler. -func (ls Labels) MarshalYAML() (interface{}, error) { - return ls.Map(), nil -} - // IsZero implements yaml.IsZeroer - if we don't have this then 'omitempty' fields are always omitted. func (ls Labels) IsZero() bool { return len(ls.data) == 0 } -// UnmarshalYAML implements yaml.Unmarshaler. -func (ls *Labels) UnmarshalYAML(unmarshal func(interface{}) error) error { - var m map[string]string - - if err := unmarshal(&m); err != nil { - return err - } - - *ls = FromMap(m) - return nil -} - // MatchLabels returns a subset of Labels that matches/does not match with the provided label names based on the 'on' boolean. // If on is set to true, it returns the subset of labels that match with the provided label names and its inverse when 'on' is set to false. // TODO: This is only used in printing an error message @@ -364,37 +291,11 @@ func (ls Labels) WithoutEmpty() Labels { return ls } -// IsValid checks if the metric name or label names are valid. -func (ls Labels) IsValid() bool { - err := ls.Validate(func(l Label) error { - if l.Name == model.MetricNameLabel && !model.IsValidMetricName(model.LabelValue(l.Value)) { - return strconv.ErrSyntax - } - if !model.LabelName(l.Name).IsValid() || !model.LabelValue(l.Value).IsValid() { - return strconv.ErrSyntax - } - return nil - }) - return err == nil -} - // Equal returns whether the two label sets are equal. func Equal(ls, o Labels) bool { return ls.data == o.data } -// Map returns a string map of the labels. -func (ls Labels) Map() map[string]string { - m := make(map[string]string, len(ls.data)/10) - for i := 0; i < len(ls.data); { - var lName, lValue string - lName, i = decodeString(ls.data, i) - lValue, i = decodeString(ls.data, i) - m[lName] = lValue - } - return m -} - // EmptyLabels returns an empty Labels value, for convenience. func EmptyLabels() Labels { return Labels{} @@ -420,15 +321,6 @@ func New(ls ...Label) Labels { return Labels{data: yoloString(buf)} } -// FromMap returns new sorted Labels from the given map. -func FromMap(m map[string]string) Labels { - l := make([]Label, 0, len(m)) - for k, v := range m { - l = append(l, Label{Name: k, Value: v}) - } - return New(l...) -} - // FromStrings creates new labels from pairs of strings. func FromStrings(ss ...string) Labels { if len(ss)%2 != 0 { @@ -547,124 +439,6 @@ func (ls Labels) ReleaseStrings(release func(string)) { release(ls.data) } -// Builder allows modifying Labels. -type Builder struct { - base Labels - del []string - add []Label -} - -// NewBuilder returns a new LabelsBuilder. -func NewBuilder(base Labels) *Builder { - b := &Builder{ - del: make([]string, 0, 5), - add: make([]Label, 0, 5), - } - b.Reset(base) - return b -} - -// Reset clears all current state for the builder. -func (b *Builder) Reset(base Labels) { - b.base = base - b.del = b.del[:0] - b.add = b.add[:0] - for i := 0; i < len(base.data); { - var lName, lValue string - lName, i = decodeString(base.data, i) - lValue, i = decodeString(base.data, i) - if lValue == "" { - b.del = append(b.del, lName) - } - } -} - -// Del deletes the label of the given name. -func (b *Builder) Del(ns ...string) *Builder { - for _, n := range ns { - for i, a := range b.add { - if a.Name == n { - b.add = append(b.add[:i], b.add[i+1:]...) - } - } - b.del = append(b.del, n) - } - return b -} - -// Keep removes all labels from the base except those with the given names. -func (b *Builder) Keep(ns ...string) *Builder { -Outer: - for i := 0; i < len(b.base.data); { - var lName string - lName, i = decodeString(b.base.data, i) - _, i = decodeString(b.base.data, i) - for _, n := range ns { - if lName == n { - continue Outer - } - } - b.del = append(b.del, lName) - } - return b -} - -// Set the name/value pair as a label. A value of "" means delete that label. -func (b *Builder) Set(n, v string) *Builder { - if v == "" { - // Empty labels are the same as missing labels. - return b.Del(n) - } - for i, a := range b.add { - if a.Name == n { - b.add[i].Value = v - return b - } - } - b.add = append(b.add, Label{Name: n, Value: v}) - - return b -} - -func (b *Builder) Get(n string) string { - // Del() removes entries from .add but Set() does not remove from .del, so check .add first. - for _, a := range b.add { - if a.Name == n { - return a.Value - } - } - if slices.Contains(b.del, n) { - return "" - } - return b.base.Get(n) -} - -// Range calls f on each label in the Builder. -func (b *Builder) Range(f func(l Label)) { - // Stack-based arrays to avoid heap allocation in most cases. - var addStack [128]Label - var delStack [128]string - // Take a copy of add and del, so they are unaffected by calls to Set() or Del(). - origAdd, origDel := append(addStack[:0], b.add...), append(delStack[:0], b.del...) - b.base.Range(func(l Label) { - if !slices.Contains(origDel, l.Name) && !contains(origAdd, l.Name) { - f(l) - } - }) - for _, a := range origAdd { - f(a) - } -} - -func contains(s []Label, n string) bool { - for _, a := range s { - if a.Name == n { - return true - } - } - return false -} - // Labels returns the labels from the builder. // If no modifications were made, the original labels are returned. func (b *Builder) Labels() Labels { @@ -829,6 +603,12 @@ func (b *ScratchBuilder) Add(name, value string) { b.add = append(b.add, Label{Name: name, Value: value}) } +// Add a name/value pair, using []byte instead of string to reduce memory allocations. +// The values must remain live until Labels() is called. +func (b *ScratchBuilder) UnsafeAddBytes(name, value []byte) { + b.add = append(b.add, Label{Name: yoloString(name), Value: yoloString(value)}) +} + // Sort the labels added so far by name. func (b *ScratchBuilder) Sort() { slices.SortFunc(b.add, func(a, b Label) int { return strings.Compare(a.Name, b.Name) }) diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/interface.go b/vendor/github.com/prometheus/prometheus/model/textparse/interface.go index 38903afc96..2f5fdbc3bf 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/interface.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/interface.go @@ -16,6 +16,8 @@ package textparse import ( "mime" + "github.com/gogo/protobuf/types" + "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -64,6 +66,11 @@ type Parser interface { // retrieved (including the case where no exemplars exist at all). Exemplar(l *exemplar.Exemplar) bool + // CreatedTimestamp writes the created timestamp of the current sample + // into the passed timestamp. It returns false if no created timestamp + // exists or if the metric type does not support created timestamps. + CreatedTimestamp(ct *types.Timestamp) bool + // Next advances the parser to the next sample. It returns false if no // more samples were read or an error occurred. Next() (Entry, error) diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go b/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go index 5623e6833f..bb50755441 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/openmetricsparse.go @@ -24,6 +24,8 @@ import ( "strings" "unicode/utf8" + "github.com/gogo/protobuf/types" + "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -211,6 +213,11 @@ func (p *OpenMetricsParser) Exemplar(e *exemplar.Exemplar) bool { return true } +// CreatedTimestamp returns false because OpenMetricsParser does not support created timestamps (yet). +func (p *OpenMetricsParser) CreatedTimestamp(_ *types.Timestamp) bool { + return false +} + // nextToken returns the next token from the openMetricsLexer. func (p *OpenMetricsParser) nextToken() token { tok := p.l.Lex() diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go b/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go index 04c295dd00..b3fa2d8a6d 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/promparse.go @@ -26,6 +26,8 @@ import ( "unicode/utf8" "unsafe" + "github.com/gogo/protobuf/types" + "github.com/prometheus/prometheus/model/exemplar" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -245,6 +247,11 @@ func (p *PromParser) Exemplar(*exemplar.Exemplar) bool { return false } +// CreatedTimestamp returns false because PromParser does not support created timestamps. +func (p *PromParser) CreatedTimestamp(_ *types.Timestamp) bool { + return false +} + // nextToken returns the next token from the promlexer. It skips over tabs // and spaces. func (p *PromParser) nextToken() token { diff --git a/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go b/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go index fbb84a2bd3..23afb5c596 100644 --- a/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go +++ b/vendor/github.com/prometheus/prometheus/model/textparse/protobufparse.go @@ -16,6 +16,7 @@ package textparse import ( "bytes" "encoding/binary" + "errors" "fmt" "io" "math" @@ -23,7 +24,7 @@ import ( "unicode/utf8" "github.com/gogo/protobuf/proto" - "github.com/pkg/errors" + "github.com/gogo/protobuf/types" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/exemplar" @@ -147,9 +148,15 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) { if ts != 0 { return p.metricBytes.Bytes(), &ts, v } - // Nasty hack: Assume that ts==0 means no timestamp. That's not true in - // general, but proto3 has no distinction between unset and - // default. Need to avoid in the final format. + // TODO(beorn7): We assume here that ts==0 means no timestamp. That's + // not true in general, but proto3 originally has no distinction between + // unset and default. At a later stage, the `optional` keyword was + // (re-)introduced in proto3, but gogo-protobuf never got updated to + // support it. (Note that setting `[(gogoproto.nullable) = true]` for + // the `timestamp_ms` field doesn't help, either.) We plan to migrate + // away from gogo-protobuf to an actively maintained protobuf + // implementation. Once that's done, we can simply use the `optional` + // keyword and check for the unset state explicitly. return p.metricBytes.Bytes(), nil, v } @@ -310,22 +317,28 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool { exProto = m.GetCounter().GetExemplar() case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: bb := m.GetHistogram().GetBucket() + isClassic := p.state == EntrySeries if p.fieldPos < 0 { - if p.state == EntrySeries { + if isClassic { return false // At _count or _sum. } p.fieldPos = 0 // Start at 1st bucket for native histograms. } for p.fieldPos < len(bb) { exProto = bb[p.fieldPos].GetExemplar() - if p.state == EntrySeries { + if isClassic { break } p.fieldPos++ - if exProto != nil { - break + // We deliberately drop exemplars with no timestamp only for native histograms. + if exProto != nil && (isClassic || exProto.GetTimestamp() != nil) { + break // Found a classic histogram exemplar or a native histogram exemplar with a timestamp. } } + // If the last exemplar for native histograms has no timestamp, ignore it. + if !isClassic && exProto.GetTimestamp() == nil { + return false + } default: return false } @@ -347,6 +360,24 @@ func (p *ProtobufParser) Exemplar(ex *exemplar.Exemplar) bool { return true } +func (p *ProtobufParser) CreatedTimestamp(ct *types.Timestamp) bool { + var foundCT *types.Timestamp + switch p.mf.GetType() { + case dto.MetricType_COUNTER: + foundCT = p.mf.GetMetric()[p.metricPos].GetCounter().GetCreatedTimestamp() + case dto.MetricType_SUMMARY: + foundCT = p.mf.GetMetric()[p.metricPos].GetSummary().GetCreatedTimestamp() + case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: + foundCT = p.mf.GetMetric()[p.metricPos].GetHistogram().GetCreatedTimestamp() + default: + } + if foundCT == nil { + return false + } + *ct = *foundCT + return true +} + // Next advances the parser to the next "sample" (emulating the behavior of a // text format parser). It returns (EntryInvalid, io.EOF) if no samples were // read. @@ -371,10 +402,10 @@ func (p *ProtobufParser) Next() (Entry, error) { // into metricBytes and validate only name, help, and type for now. name := p.mf.GetName() if !model.IsValidMetricName(model.LabelValue(name)) { - return EntryInvalid, errors.Errorf("invalid metric name: %s", name) + return EntryInvalid, fmt.Errorf("invalid metric name: %s", name) } if help := p.mf.GetHelp(); !utf8.ValidString(help) { - return EntryInvalid, errors.Errorf("invalid help for metric %q: %s", name, help) + return EntryInvalid, fmt.Errorf("invalid help for metric %q: %s", name, help) } switch p.mf.GetType() { case dto.MetricType_COUNTER, @@ -385,7 +416,7 @@ func (p *ProtobufParser) Next() (Entry, error) { dto.MetricType_UNTYPED: // All good. default: - return EntryInvalid, errors.Errorf("unknown metric type for metric %q: %s", name, p.mf.GetType()) + return EntryInvalid, fmt.Errorf("unknown metric type for metric %q: %s", name, p.mf.GetType()) } p.metricBytes.Reset() p.metricBytes.WriteString(name) @@ -438,7 +469,7 @@ func (p *ProtobufParser) Next() (Entry, error) { return EntryInvalid, err } default: - return EntryInvalid, errors.Errorf("invalid protobuf parsing state: %d", p.state) + return EntryInvalid, fmt.Errorf("invalid protobuf parsing state: %d", p.state) } return p.state, nil } @@ -451,13 +482,13 @@ func (p *ProtobufParser) updateMetricBytes() error { b.WriteByte(model.SeparatorByte) n := lp.GetName() if !model.LabelName(n).IsValid() { - return errors.Errorf("invalid label name: %s", n) + return fmt.Errorf("invalid label name: %s", n) } b.WriteString(n) b.WriteByte(model.SeparatorByte) v := lp.GetValue() if !utf8.ValidString(v) { - return errors.Errorf("invalid label value: %s", v) + return fmt.Errorf("invalid label value: %s", v) } b.WriteString(v) } @@ -532,7 +563,7 @@ func readDelimited(b []byte, mf *dto.MetricFamily) (n int, err error) { } totalLength := varIntLength + int(messageLength) if totalLength > len(b) { - return 0, errors.Errorf("protobufparse: insufficient length of buffer, expected at least %d bytes, got %d bytes", totalLength, len(b)) + return 0, fmt.Errorf("protobufparse: insufficient length of buffer, expected at least %d bytes, got %d bytes", totalLength, len(b)) } mf.Reset() return totalLength, mf.Unmarshal(b[varIntLength:totalLength]) diff --git a/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.pb.go b/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.pb.go index 05d25747b4..e6623e9e19 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.pb.go +++ b/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.pb.go @@ -411,7 +411,7 @@ type Histogram struct { SampleCount uint64 `protobuf:"varint,1,opt,name=sample_count,json=sampleCount,proto3" json:"sample_count,omitempty"` SampleCountFloat float64 `protobuf:"fixed64,4,opt,name=sample_count_float,json=sampleCountFloat,proto3" json:"sample_count_float,omitempty"` SampleSum float64 `protobuf:"fixed64,2,opt,name=sample_sum,json=sampleSum,proto3" json:"sample_sum,omitempty"` - // Buckets for the conventional histogram. + // Buckets for the classic histogram. Bucket []Bucket `protobuf:"bytes,3,rep,name=bucket,proto3" json:"bucket"` CreatedTimestamp *types.Timestamp `protobuf:"bytes,15,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` // schema defines the bucket schema. Currently, valid numbers are -4 <= n <= 8. @@ -965,68 +965,67 @@ func init() { } var fileDescriptor_d1e5ddb18987a258 = []byte{ - // 963 bytes of a gzipped FileDescriptorProto + // 960 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0xee, 0x76, 0xfd, 0x93, 0x3d, 0x8e, 0x93, 0xcd, 0x60, 0x55, 0xab, 0x40, 0x62, 0xb3, 0x12, - 0x52, 0x40, 0xc8, 0x16, 0x50, 0x04, 0x2a, 0x45, 0x22, 0x69, 0xd3, 0x14, 0x15, 0xb7, 0x65, 0x6c, - 0x5f, 0x94, 0x9b, 0xd5, 0xd8, 0x9e, 0xac, 0x57, 0xec, 0xee, 0x2c, 0xfb, 0x53, 0x11, 0xee, 0x79, - 0x06, 0x5e, 0x01, 0xf1, 0x1c, 0x08, 0xf5, 0x92, 0x07, 0x40, 0x08, 0xe5, 0x49, 0xd0, 0xfc, 0xed, - 0x3a, 0xd5, 0xba, 0x90, 0xf6, 0x6e, 0xe6, 0xf3, 0x77, 0xce, 0x7c, 0xe7, 0x9b, 0xf1, 0x39, 0x0b, - 0x6e, 0xc0, 0x46, 0x49, 0xca, 0x22, 0x9a, 0xaf, 0x68, 0x91, 0x8d, 0x16, 0x61, 0x40, 0xe3, 0x7c, - 0x14, 0xd1, 0x3c, 0x0d, 0x16, 0xd9, 0x30, 0x49, 0x59, 0xce, 0x50, 0x2f, 0x60, 0xc3, 0x8a, 0x33, - 0x94, 0x9c, 0xfd, 0x9e, 0xcf, 0x7c, 0x26, 0x08, 0x23, 0xbe, 0x92, 0xdc, 0xfd, 0xbe, 0xcf, 0x98, - 0x1f, 0xd2, 0x91, 0xd8, 0xcd, 0x8b, 0xf3, 0x51, 0x1e, 0x44, 0x34, 0xcb, 0x49, 0x94, 0x48, 0x82, - 0xfb, 0x29, 0x58, 0xdf, 0x90, 0x39, 0x0d, 0x9f, 0x92, 0x20, 0x45, 0x08, 0x1a, 0x31, 0x89, 0xa8, - 0x63, 0x0c, 0x8c, 0x23, 0x0b, 0x8b, 0x35, 0xea, 0x41, 0xf3, 0x39, 0x09, 0x0b, 0xea, 0xdc, 0x14, - 0xa0, 0xdc, 0xb8, 0x07, 0xd0, 0x3c, 0x23, 0x85, 0xbf, 0xf6, 0x33, 0x8f, 0x31, 0xf4, 0xcf, 0xbf, - 0x19, 0xd0, 0xbe, 0xc7, 0x8a, 0x38, 0xa7, 0x69, 0x3d, 0x03, 0xdd, 0x81, 0x2d, 0xfa, 0x23, 0x8d, - 0x92, 0x90, 0xa4, 0x22, 0x73, 0xe7, 0xe3, 0xc3, 0x61, 0x5d, 0x5d, 0xc3, 0x53, 0xc5, 0xc2, 0x25, - 0x1f, 0x8d, 0x61, 0x6f, 0x91, 0x52, 0x92, 0xd3, 0xa5, 0x57, 0x96, 0xe3, 0x98, 0x22, 0xc9, 0xfe, - 0x50, 0x16, 0x3c, 0xd4, 0x05, 0x0f, 0xa7, 0x9a, 0x71, 0xd2, 0x78, 0xf1, 0x77, 0xdf, 0xc0, 0xb6, - 0x0a, 0x2d, 0x71, 0xf7, 0x2e, 0x6c, 0x7d, 0x5b, 0x90, 0x38, 0x0f, 0x42, 0x8a, 0xf6, 0x61, 0xeb, - 0x07, 0xb5, 0x56, 0x7a, 0xcb, 0xfd, 0x55, 0x27, 0xca, 0x52, 0xff, 0x32, 0xa0, 0x3d, 0x29, 0xa2, - 0x88, 0xa4, 0x17, 0xe8, 0x5d, 0xd8, 0xce, 0x48, 0x94, 0x84, 0xd4, 0x5b, 0xf0, 0xe2, 0x45, 0x86, - 0x06, 0xee, 0x48, 0x4c, 0xf8, 0x81, 0x0e, 0x00, 0x14, 0x25, 0x2b, 0x22, 0x95, 0xc9, 0x92, 0xc8, - 0xa4, 0x88, 0xd0, 0x57, 0x6b, 0xe7, 0x9b, 0x03, 0x73, 0xb3, 0x2d, 0x5a, 0xb1, 0xa8, 0xea, 0xc6, - 0x9a, 0xca, 0x5a, 0x73, 0x1a, 0xaf, 0x6d, 0x4e, 0x1f, 0xda, 0xb3, 0x38, 0xbf, 0x48, 0xe8, 0x72, - 0xc3, 0x55, 0xff, 0xde, 0x04, 0xeb, 0x61, 0x90, 0xe5, 0xcc, 0x4f, 0x49, 0xf4, 0x7f, 0x1c, 0xf8, - 0x10, 0xd0, 0x3a, 0xc5, 0x3b, 0x0f, 0x19, 0xc9, 0x85, 0x42, 0x03, 0xdb, 0x6b, 0xc4, 0x07, 0x1c, - 0xff, 0x2f, 0xbf, 0xee, 0x40, 0x6b, 0x5e, 0x2c, 0xbe, 0xa7, 0xb9, 0x72, 0xeb, 0x9d, 0x7a, 0xb7, - 0x4e, 0x04, 0x47, 0x79, 0xa5, 0x22, 0xea, 0x9d, 0xda, 0x7d, 0x5d, 0xa7, 0xd0, 0x2d, 0x68, 0x65, - 0x8b, 0x15, 0x8d, 0x88, 0xd3, 0x1c, 0x18, 0x47, 0x7b, 0x58, 0xed, 0xd0, 0x7b, 0xb0, 0xf3, 0x13, - 0x4d, 0x99, 0x97, 0xaf, 0x52, 0x9a, 0xad, 0x58, 0xb8, 0x74, 0x5a, 0xa2, 0x8a, 0x2e, 0x47, 0xa7, - 0x1a, 0xe4, 0x85, 0x0a, 0x9a, 0xf4, 0xad, 0x2d, 0x7c, 0xb3, 0x38, 0x22, 0x5d, 0x3b, 0x02, 0xbb, - 0xfa, 0x59, 0x79, 0xb6, 0x25, 0xf2, 0xec, 0x94, 0x24, 0xe9, 0xd8, 0x23, 0xe8, 0xc6, 0xd4, 0x27, - 0x79, 0xf0, 0x9c, 0x7a, 0x59, 0x42, 0x62, 0xc7, 0x12, 0xce, 0x0c, 0x5e, 0xe5, 0xcc, 0x24, 0x21, - 0xb1, 0x72, 0x67, 0x5b, 0x07, 0x73, 0x8c, 0x8b, 0x2f, 0x93, 0x2d, 0x69, 0x98, 0x13, 0x07, 0x06, - 0xe6, 0x11, 0xc2, 0xe5, 0x11, 0xf7, 0x39, 0x78, 0x85, 0x26, 0x0b, 0xe8, 0x0c, 0x4c, 0x5e, 0xa3, - 0x46, 0x65, 0x11, 0x8f, 0xa0, 0x9b, 0xb0, 0x2c, 0xa8, 0xa4, 0x6d, 0x5f, 0x4f, 0x9a, 0x0e, 0xd6, - 0xd2, 0xca, 0x64, 0x52, 0x5a, 0x57, 0x4a, 0xd3, 0x68, 0x29, 0xad, 0xa4, 0x49, 0x69, 0x3b, 0x52, - 0x9a, 0x46, 0x85, 0x34, 0xf7, 0x0f, 0x03, 0x5a, 0xf2, 0x40, 0xf4, 0x3e, 0xd8, 0x8b, 0x22, 0x2a, - 0xc2, 0xf5, 0x72, 0xe4, 0x3b, 0xde, 0xad, 0x70, 0x59, 0xd0, 0x6d, 0xb8, 0xf5, 0x32, 0xf5, 0xca, - 0x7b, 0xee, 0xbd, 0x14, 0x20, 0x6f, 0xa8, 0x0f, 0x9d, 0x22, 0x49, 0x68, 0xea, 0xcd, 0x59, 0x11, - 0x2f, 0xd5, 0xa3, 0x06, 0x01, 0x9d, 0x70, 0xe4, 0x4a, 0x73, 0x34, 0xaf, 0xd7, 0x1c, 0xdd, 0xbb, - 0x00, 0x95, 0x71, 0xfc, 0x51, 0xb2, 0xf3, 0xf3, 0x8c, 0xca, 0x0a, 0xf6, 0xb0, 0xda, 0x71, 0x3c, - 0xa4, 0xb1, 0x9f, 0xaf, 0xc4, 0xe9, 0x5d, 0xac, 0x76, 0xee, 0x2f, 0x06, 0x6c, 0xe9, 0xa4, 0xe8, - 0x0b, 0x68, 0x86, 0x7c, 0x36, 0x38, 0x86, 0xb8, 0xa6, 0x7e, 0xbd, 0x86, 0x72, 0x7c, 0xa8, 0x5b, - 0x92, 0x31, 0xf5, 0xdd, 0x12, 0x7d, 0x0e, 0xd6, 0x35, 0x5a, 0x36, 0xae, 0xc8, 0xee, 0xcf, 0x26, - 0xb4, 0xc6, 0x62, 0x0e, 0xbe, 0x99, 0xae, 0x8f, 0xa0, 0xe9, 0xf3, 0xc9, 0xa5, 0xa6, 0xce, 0xdb, - 0xf5, 0xc1, 0x62, 0xb8, 0x61, 0xc9, 0x44, 0x9f, 0x41, 0x7b, 0x21, 0x87, 0x99, 0x92, 0x7c, 0x50, - 0x1f, 0xa4, 0x26, 0x1e, 0xd6, 0x6c, 0x1e, 0x98, 0xc9, 0xd1, 0xa0, 0x3a, 0xf0, 0x86, 0x40, 0x35, - 0x3f, 0xb0, 0x66, 0xf3, 0xc0, 0x42, 0x76, 0x5d, 0xd1, 0x4c, 0x36, 0x06, 0xaa, 0xd6, 0x8c, 0x35, - 0x1b, 0x7d, 0x09, 0xd6, 0x4a, 0x37, 0x63, 0xd1, 0x44, 0x36, 0xda, 0x53, 0xf6, 0x6c, 0x5c, 0x45, - 0xf0, 0xf6, 0x5d, 0x3a, 0xee, 0x45, 0x99, 0xe8, 0x54, 0x26, 0xee, 0x94, 0xd8, 0x38, 0x73, 0x7f, - 0x35, 0x60, 0x5b, 0xde, 0xc3, 0x03, 0x12, 0x05, 0xe1, 0x45, 0xed, 0x47, 0x03, 0x82, 0xc6, 0x8a, - 0x86, 0x89, 0xfa, 0x66, 0x10, 0x6b, 0x74, 0x1b, 0x1a, 0x5c, 0xa3, 0xb0, 0x70, 0x67, 0xd3, 0x7f, - 0x5e, 0x66, 0x9e, 0x5e, 0x24, 0x14, 0x0b, 0x36, 0x6f, 0xf0, 0xf2, 0xeb, 0xc7, 0x69, 0xbc, 0xaa, - 0xc1, 0xcb, 0x38, 0xdd, 0xe0, 0x65, 0xc4, 0x07, 0x73, 0x80, 0x2a, 0x1f, 0xea, 0x40, 0xfb, 0xde, - 0x93, 0xd9, 0xe3, 0xe9, 0x29, 0xb6, 0x6f, 0x20, 0x0b, 0x9a, 0x67, 0xc7, 0xb3, 0xb3, 0x53, 0xdb, - 0xe0, 0xf8, 0x64, 0x36, 0x1e, 0x1f, 0xe3, 0x67, 0xf6, 0x4d, 0xbe, 0x99, 0x3d, 0x9e, 0x3e, 0x7b, - 0x7a, 0x7a, 0xdf, 0x36, 0x51, 0x17, 0xac, 0x87, 0x5f, 0x4f, 0xa6, 0x4f, 0xce, 0xf0, 0xf1, 0xd8, - 0x6e, 0xa0, 0xb7, 0x60, 0x57, 0xc4, 0x78, 0x15, 0xd8, 0x3c, 0x71, 0x5f, 0x5c, 0x1e, 0x1a, 0x7f, - 0x5e, 0x1e, 0x1a, 0xff, 0x5c, 0x1e, 0x1a, 0xdf, 0xf5, 0x02, 0xe6, 0x55, 0xe2, 0x3c, 0x29, 0x6e, - 0xde, 0x12, 0x2f, 0xfb, 0x93, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x68, 0x3f, 0xd9, 0x07, 0xdd, - 0x09, 0x00, 0x00, + 0x14, 0xee, 0xd6, 0xbf, 0x7b, 0x1c, 0x27, 0x9b, 0xc1, 0xaa, 0x56, 0x81, 0xc4, 0x66, 0x25, 0xa4, + 0x80, 0x90, 0x2d, 0xa0, 0x08, 0x54, 0x8a, 0x44, 0xd2, 0xa6, 0x2e, 0x2a, 0x6e, 0xcb, 0xd8, 0xbe, + 0x28, 0x37, 0xab, 0xb1, 0x3d, 0x59, 0xaf, 0xd8, 0xdd, 0x59, 0xf6, 0xa7, 0x22, 0xdc, 0xf3, 0x0c, + 0xbc, 0x00, 0x17, 0x3c, 0x05, 0x97, 0xa8, 0x97, 0x5c, 0x71, 0x89, 0x50, 0x9e, 0x04, 0xcd, 0xdf, + 0xae, 0x53, 0xad, 0x03, 0x81, 0xbb, 0x99, 0xcf, 0xdf, 0x39, 0xf3, 0x9d, 0x6f, 0xc6, 0xe7, 0x2c, + 0x38, 0x3e, 0x1b, 0xc5, 0x09, 0x0b, 0x69, 0xb6, 0xa6, 0x79, 0x3a, 0x5a, 0x06, 0x3e, 0x8d, 0xb2, + 0x51, 0x48, 0xb3, 0xc4, 0x5f, 0xa6, 0xc3, 0x38, 0x61, 0x19, 0x43, 0x3d, 0x9f, 0x0d, 0x4b, 0xce, + 0x50, 0x72, 0x0e, 0x7a, 0x1e, 0xf3, 0x98, 0x20, 0x8c, 0xf8, 0x4a, 0x72, 0x0f, 0xfa, 0x1e, 0x63, + 0x5e, 0x40, 0x47, 0x62, 0xb7, 0xc8, 0xcf, 0x47, 0x99, 0x1f, 0xd2, 0x34, 0x23, 0x61, 0x2c, 0x09, + 0xce, 0xc7, 0x60, 0x7e, 0x45, 0x16, 0x34, 0x78, 0x4e, 0xfc, 0x04, 0x21, 0xa8, 0x47, 0x24, 0xa4, + 0xb6, 0x31, 0x30, 0x8e, 0x4d, 0x2c, 0xd6, 0xa8, 0x07, 0x8d, 0x97, 0x24, 0xc8, 0xa9, 0x7d, 0x5b, + 0x80, 0x72, 0xe3, 0x1c, 0x42, 0x63, 0x4c, 0x72, 0x6f, 0xe3, 0x67, 0x1e, 0x63, 0xe8, 0x9f, 0x7f, + 0x36, 0xa0, 0xf5, 0x80, 0xe5, 0x51, 0x46, 0x93, 0x6a, 0x06, 0xba, 0x07, 0x6d, 0xfa, 0x3d, 0x0d, + 0xe3, 0x80, 0x24, 0x22, 0x73, 0xe7, 0xc3, 0xa3, 0x61, 0x55, 0x5d, 0xc3, 0x33, 0xc5, 0xc2, 0x05, + 0x1f, 0x8d, 0x61, 0x7f, 0x99, 0x50, 0x92, 0xd1, 0x95, 0x5b, 0x94, 0x63, 0xd7, 0x44, 0x92, 0x83, + 0xa1, 0x2c, 0x78, 0xa8, 0x0b, 0x1e, 0xce, 0x34, 0x03, 0x5b, 0x2a, 0xa8, 0x40, 0x9c, 0xfb, 0xd0, + 0xfe, 0x3a, 0x27, 0x51, 0xe6, 0x07, 0x14, 0x1d, 0x40, 0xfb, 0x3b, 0xb5, 0x56, 0x4a, 0x8b, 0xfd, + 0x55, 0x0f, 0x8a, 0x22, 0xff, 0x30, 0xa0, 0x35, 0xcd, 0xc3, 0x90, 0x24, 0x17, 0xe8, 0x6d, 0xd8, + 0x49, 0x49, 0x18, 0x07, 0xd4, 0x5d, 0xf2, 0xb2, 0x45, 0x86, 0x3a, 0xee, 0x48, 0x4c, 0x38, 0x81, + 0x0e, 0x01, 0x14, 0x25, 0xcd, 0x43, 0x95, 0xc9, 0x94, 0xc8, 0x34, 0x0f, 0xd1, 0x17, 0x1b, 0xe7, + 0xd7, 0x06, 0xb5, 0xed, 0x86, 0x68, 0xc5, 0xa7, 0xf5, 0x57, 0x7f, 0xf6, 0x6f, 0x6d, 0xa8, 0xac, + 0xb4, 0xa5, 0xfe, 0x1f, 0x6c, 0xe9, 0x43, 0x6b, 0x1e, 0x65, 0x17, 0x31, 0x5d, 0x6d, 0xb9, 0xde, + 0x5f, 0x1b, 0x60, 0x3e, 0xf6, 0xd3, 0x8c, 0x79, 0x09, 0x09, 0xff, 0x4d, 0xed, 0xef, 0x03, 0xda, + 0xa4, 0xb8, 0xe7, 0x01, 0x23, 0x99, 0xd0, 0x66, 0x60, 0x6b, 0x83, 0xf8, 0x88, 0xe3, 0xff, 0xe4, + 0xd4, 0x3d, 0x68, 0x2e, 0xf2, 0xe5, 0xb7, 0x34, 0x53, 0x3e, 0xbd, 0x55, 0xed, 0xd3, 0xa9, 0xe0, + 0x28, 0x97, 0x54, 0x44, 0xb5, 0x47, 0x7b, 0x37, 0xf7, 0x08, 0xdd, 0x81, 0x66, 0xba, 0x5c, 0xd3, + 0x90, 0xd8, 0x8d, 0x81, 0x71, 0xbc, 0x8f, 0xd5, 0x0e, 0xbd, 0x03, 0xbb, 0x3f, 0xd0, 0x84, 0xb9, + 0xd9, 0x3a, 0xa1, 0xe9, 0x9a, 0x05, 0x2b, 0xbb, 0x29, 0xf4, 0x77, 0x39, 0x3a, 0xd3, 0x20, 0x2f, + 0x51, 0xd0, 0xa4, 0x63, 0x2d, 0xe1, 0x98, 0xc9, 0x11, 0xe9, 0xd7, 0x31, 0x58, 0xe5, 0xcf, 0xca, + 0xad, 0xb6, 0xc8, 0xb3, 0x5b, 0x90, 0xa4, 0x57, 0x4f, 0xa0, 0x1b, 0x51, 0x8f, 0x64, 0xfe, 0x4b, + 0xea, 0xa6, 0x31, 0x89, 0x6c, 0x53, 0x78, 0x32, 0xb8, 0xce, 0x93, 0x69, 0x4c, 0x22, 0xe5, 0xcb, + 0x8e, 0x0e, 0xe6, 0x18, 0x17, 0x5f, 0x24, 0x5b, 0xd1, 0x20, 0x23, 0x36, 0x0c, 0x6a, 0xc7, 0x08, + 0x17, 0x47, 0x3c, 0xe4, 0xe0, 0x15, 0x9a, 0x2c, 0xa0, 0x33, 0xa8, 0xf1, 0x1a, 0x35, 0x2a, 0x8b, + 0x78, 0x02, 0xdd, 0x98, 0xa5, 0x7e, 0x29, 0x6d, 0xe7, 0x66, 0xd2, 0x74, 0xb0, 0x96, 0x56, 0x24, + 0x93, 0xd2, 0xba, 0x52, 0x9a, 0x46, 0x0b, 0x69, 0x05, 0x4d, 0x4a, 0xdb, 0x95, 0xd2, 0x34, 0x2a, + 0xa4, 0x39, 0xbf, 0x19, 0xd0, 0x94, 0x07, 0xa2, 0x77, 0xc1, 0x5a, 0xe6, 0x61, 0x1e, 0x6c, 0x96, + 0x23, 0x5f, 0xf0, 0x5e, 0x89, 0xcb, 0x82, 0xee, 0xc2, 0x9d, 0xd7, 0xa9, 0x57, 0x5e, 0x72, 0xef, + 0xb5, 0x00, 0x79, 0x43, 0x7d, 0xe8, 0xe4, 0x71, 0x4c, 0x13, 0x77, 0xc1, 0xf2, 0x68, 0xa5, 0x9e, + 0x33, 0x08, 0xe8, 0x94, 0x23, 0x57, 0x5a, 0x61, 0xed, 0x66, 0xad, 0xd0, 0xb9, 0x0f, 0x50, 0x1a, + 0xc7, 0x1f, 0x25, 0x3b, 0x3f, 0x4f, 0xa9, 0xac, 0x60, 0x1f, 0xab, 0x1d, 0xc7, 0x03, 0x1a, 0x79, + 0xd9, 0x5a, 0x9c, 0xde, 0xc5, 0x6a, 0xe7, 0xfc, 0x64, 0x40, 0x5b, 0x27, 0x45, 0x9f, 0x41, 0x23, + 0xe0, 0x93, 0xc0, 0x36, 0xc4, 0x35, 0xf5, 0xab, 0x35, 0x14, 0xc3, 0x42, 0xdd, 0x92, 0x8c, 0xa9, + 0xee, 0x90, 0xe8, 0x53, 0x30, 0x6f, 0xd2, 0xa0, 0x4b, 0xb2, 0xf3, 0x63, 0x0d, 0x9a, 0x13, 0x31, + 0xf5, 0xfe, 0x9f, 0xae, 0x0f, 0xa0, 0xe1, 0xf1, 0x39, 0xa5, 0x66, 0xcc, 0x9b, 0xd5, 0xc1, 0x62, + 0x94, 0x61, 0xc9, 0x44, 0x9f, 0x40, 0x6b, 0x29, 0x47, 0x97, 0x92, 0x7c, 0x58, 0x1d, 0xa4, 0xe6, + 0x1b, 0xd6, 0x6c, 0x1e, 0x98, 0xca, 0x71, 0xa0, 0xba, 0xee, 0x96, 0x40, 0x35, 0x33, 0xb0, 0x66, + 0xf3, 0xc0, 0x5c, 0xf6, 0x5b, 0xd1, 0x4c, 0xb6, 0x06, 0xaa, 0xa6, 0x8c, 0x35, 0x1b, 0x7d, 0x0e, + 0xe6, 0x5a, 0xb7, 0x61, 0xd1, 0x44, 0xb6, 0xda, 0x53, 0x74, 0x6b, 0x5c, 0x46, 0xf0, 0xc6, 0x5d, + 0x38, 0xee, 0x86, 0xa9, 0xe8, 0x54, 0x35, 0xdc, 0x29, 0xb0, 0x49, 0xea, 0xfc, 0x62, 0xc0, 0x8e, + 0xbc, 0x87, 0x47, 0x24, 0xf4, 0x83, 0x8b, 0xca, 0x4f, 0x04, 0x04, 0xf5, 0x35, 0x0d, 0x62, 0xf5, + 0x85, 0x20, 0xd6, 0xe8, 0x2e, 0xd4, 0xb9, 0x46, 0x61, 0xe1, 0xee, 0xb6, 0xff, 0xbc, 0xcc, 0x3c, + 0xbb, 0x88, 0x29, 0x16, 0x6c, 0xde, 0xda, 0xe5, 0xb7, 0x8e, 0x5d, 0xbf, 0xae, 0xb5, 0xcb, 0x38, + 0xdd, 0xda, 0x65, 0xc4, 0x7b, 0x0b, 0x80, 0x32, 0x1f, 0xea, 0x40, 0xeb, 0xc1, 0xb3, 0xf9, 0xd3, + 0xd9, 0x19, 0xb6, 0x6e, 0x21, 0x13, 0x1a, 0xe3, 0x93, 0xf9, 0xf8, 0xcc, 0x32, 0x38, 0x3e, 0x9d, + 0x4f, 0x26, 0x27, 0xf8, 0x85, 0x75, 0x9b, 0x6f, 0xe6, 0x4f, 0x67, 0x2f, 0x9e, 0x9f, 0x3d, 0xb4, + 0x6a, 0xa8, 0x0b, 0xe6, 0xe3, 0x2f, 0xa7, 0xb3, 0x67, 0x63, 0x7c, 0x32, 0xb1, 0xea, 0xe8, 0x0d, + 0xd8, 0x13, 0x31, 0x6e, 0x09, 0x36, 0x4e, 0x9d, 0x57, 0x97, 0x47, 0xc6, 0xef, 0x97, 0x47, 0xc6, + 0x5f, 0x97, 0x47, 0xc6, 0x37, 0x3d, 0x9f, 0xb9, 0xa5, 0x38, 0x57, 0x8a, 0x5b, 0x34, 0xc5, 0xcb, + 0xfe, 0xe8, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x2e, 0x66, 0xc1, 0xcb, 0x09, 0x00, 0x00, } func (m *LabelPair) Marshal() (dAtA []byte, err error) { diff --git a/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.proto b/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.proto index 8e225bb3b9..13ac8dcb4d 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.proto +++ b/vendor/github.com/prometheus/prometheus/prompb/io/prometheus/client/metrics.proto @@ -52,7 +52,7 @@ message Counter { double value = 1; Exemplar exemplar = 2; - google.protobuf.Timestamp created_timestamp = 3 [(gogoproto.nullable) = true]; + google.protobuf.Timestamp created_timestamp = 3; } message Quantile { @@ -65,7 +65,7 @@ message Summary { double sample_sum = 2; repeated Quantile quantile = 3 [(gogoproto.nullable) = false]; - google.protobuf.Timestamp created_timestamp = 4 [(gogoproto.nullable) = true]; + google.protobuf.Timestamp created_timestamp = 4; } message Untyped { @@ -76,10 +76,10 @@ message Histogram { uint64 sample_count = 1; double sample_count_float = 4; // Overrides sample_count if > 0. double sample_sum = 2; - // Buckets for the conventional histogram. + // Buckets for the classic histogram. repeated Bucket bucket = 3 [(gogoproto.nullable) = false]; // Ordered in increasing order of upper_bound, +Inf bucket is optional. - google.protobuf.Timestamp created_timestamp = 15 [(gogoproto.nullable) = true]; + google.protobuf.Timestamp created_timestamp = 15; // Everything below here is for native histograms (also known as sparse histograms). // Native histograms are an experimental feature without stability guarantees. @@ -153,4 +153,4 @@ message MetricFamily { string help = 2; MetricType type = 3; repeated Metric metric = 4 [(gogoproto.nullable) = false]; -} \ No newline at end of file +} diff --git a/vendor/github.com/prometheus/prometheus/promql/engine.go b/vendor/github.com/prometheus/prometheus/promql/engine.go index 4238495675..4fa2a513be 100644 --- a/vendor/github.com/prometheus/prometheus/promql/engine.go +++ b/vendor/github.com/prometheus/prometheus/promql/engine.go @@ -674,7 +674,7 @@ func durationMilliseconds(d time.Duration) int64 { // execEvalStmt evaluates the expression of an evaluation statement for the given time range. func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.EvalStmt) (parser.Value, annotations.Annotations, error) { prepareSpanTimer, ctxPrepare := query.stats.GetSpanTimer(ctx, stats.QueryPreparationTime, ng.metrics.queryPrepareTime) - mint, maxt := ng.findMinMaxTime(s) + mint, maxt := FindMinMaxTime(s) querier, err := query.queryable.Querier(mint, maxt) if err != nil { prepareSpanTimer.Finish() @@ -816,7 +816,10 @@ func subqueryTimes(path []parser.Node) (time.Duration, time.Duration, *int64) { return subqOffset, subqRange, tsp } -func (ng *Engine) findMinMaxTime(s *parser.EvalStmt) (int64, int64) { +// FindMinMaxTime returns the time in milliseconds of the earliest and latest point in time the statement will try to process. +// This takes into account offsets, @ modifiers, and range selectors. +// If the statement does not select series, then FindMinMaxTime returns (0, 0). +func FindMinMaxTime(s *parser.EvalStmt) (int64, int64) { var minTimestamp, maxTimestamp int64 = math.MaxInt64, math.MinInt64 // Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range. // The evaluation of the VectorSelector inside then evaluates the given range and unsets @@ -825,7 +828,7 @@ func (ng *Engine) findMinMaxTime(s *parser.EvalStmt) (int64, int64) { parser.Inspect(s.Expr, func(node parser.Node, path []parser.Node) error { switch n := node.(type) { case *parser.VectorSelector: - start, end := ng.getTimeRangesForSelector(s, n, path, evalRange) + start, end := getTimeRangesForSelector(s, n, path, evalRange) if start < minTimestamp { minTimestamp = start } @@ -848,7 +851,7 @@ func (ng *Engine) findMinMaxTime(s *parser.EvalStmt) (int64, int64) { return minTimestamp, maxTimestamp } -func (ng *Engine) getTimeRangesForSelector(s *parser.EvalStmt, n *parser.VectorSelector, path []parser.Node, evalRange time.Duration) (int64, int64) { +func getTimeRangesForSelector(s *parser.EvalStmt, n *parser.VectorSelector, path []parser.Node, evalRange time.Duration) (int64, int64) { start, end := timestamp.FromTime(s.Start), timestamp.FromTime(s.End) subqOffset, subqRange, subqTs := subqueryTimes(path) @@ -905,7 +908,7 @@ func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s parser.Inspect(s.Expr, func(node parser.Node, path []parser.Node) error { switch n := node.(type) { case *parser.VectorSelector: - start, end := ng.getTimeRangesForSelector(s, n, path, evalRange) + start, end := getTimeRangesForSelector(s, n, path, evalRange) interval := ng.getLastSubqueryInterval(path) if interval == 0 { interval = s.Interval @@ -1071,7 +1074,7 @@ type EvalNodeHelper struct { // Caches. // DropMetricName and label_*. Dmn map[uint64]labels.Labels - // funcHistogramQuantile for conventional histograms. + // funcHistogramQuantile for classic histograms. signatureToMetricWithBuckets map[string]*metricWithBuckets // label_replace. regex *regexp.Regexp @@ -1172,9 +1175,7 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper) bufHelpers[i] = make([]EvalSeriesHelper, len(matrixes[i])) for si, series := range matrixes[i] { - h := seriesHelpers[i][si] - prepSeries(series.Metric, &h) - seriesHelpers[i][si] = h + prepSeries(series.Metric, &seriesHelpers[i][si]) } } } @@ -1224,10 +1225,11 @@ func (ev *evaluator) rangeEval(prepSeries func(labels.Labels, *EvalSeriesHelper) enh.Out = result[:0] // Reuse result vector. warnings.Merge(ws) - ev.currentSamples += len(result) + vecNumSamples := result.TotalSamples() + ev.currentSamples += vecNumSamples // When we reset currentSamples to tempNumSamples during the next iteration of the loop it also // needs to include the samples from the result here, as they're still in memory. - tempNumSamples += len(result) + tempNumSamples += vecNumSamples ev.samplesStats.UpdatePeak(ev.currentSamples) if ev.currentSamples > ev.maxSamples { @@ -1323,12 +1325,10 @@ func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) (*parser.MatrixSele Range: subq.Range, VectorSelector: vs, } - totalSamples := 0 for _, s := range mat { - totalSamples += len(s.Floats) + len(s.Histograms) vs.Series = append(vs.Series, NewStorageSeries(s)) } - return ms, totalSamples, ws + return ms, mat.TotalSamples(), ws } // eval evaluates the given expression as the given AST expression node requires. @@ -1470,7 +1470,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio it := storage.NewBuffer(selRange) var chkIter chunkenc.Iterator for i, s := range selVS.Series { - ev.currentSamples -= len(floats) + len(histograms) + ev.currentSamples -= len(floats) + totalHPointSize(histograms) if floats != nil { floats = floats[:0] } @@ -1514,7 +1514,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio // Make the function call. outVec, annos := call(inArgs, e.Args, enh) warnings.Merge(annos) - ev.samplesStats.IncrementSamplesAtStep(step, int64(len(floats)+len(histograms))) + ev.samplesStats.IncrementSamplesAtStep(step, int64(len(floats)+totalHPointSize(histograms))) enh.Out = outVec[:0] if len(outVec) > 0 { @@ -1533,10 +1533,11 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio // Only buffer stepRange milliseconds from the second step on. it.ReduceDelta(stepRange) } - if len(ss.Floats)+len(ss.Histograms) > 0 { - if ev.currentSamples+len(ss.Floats)+len(ss.Histograms) <= ev.maxSamples { + histSamples := totalHPointSize(ss.Histograms) + if len(ss.Floats)+histSamples > 0 { + if ev.currentSamples+len(ss.Floats)+histSamples <= ev.maxSamples { mat = append(mat, ss) - ev.currentSamples += len(ss.Floats) + len(ss.Histograms) + ev.currentSamples += len(ss.Floats) + histSamples } else { ev.error(ErrTooManySamples(env)) } @@ -1545,7 +1546,7 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio } ev.samplesStats.UpdatePeak(ev.currentSamples) - ev.currentSamples -= len(floats) + len(histograms) + ev.currentSamples -= len(floats) + totalHPointSize(histograms) putFPointSlice(floats) putHPointSlice(histograms) @@ -1692,14 +1693,18 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio ss.Floats = getFPointSlice(numSteps) } ss.Floats = append(ss.Floats, FPoint{F: f, T: ts}) + ev.currentSamples++ + ev.samplesStats.IncrementSamplesAtStep(step, 1) } else { if ss.Histograms == nil { ss.Histograms = getHPointSlice(numSteps) } - ss.Histograms = append(ss.Histograms, HPoint{H: h, T: ts}) + point := HPoint{H: h, T: ts} + ss.Histograms = append(ss.Histograms, point) + histSize := point.size() + ev.currentSamples += histSize + ev.samplesStats.IncrementSamplesAtStep(step, int64(histSize)) } - ev.samplesStats.IncrementSamplesAtStep(step, 1) - ev.currentSamples++ } else { ev.error(ErrTooManySamples(env)) } @@ -1807,13 +1812,15 @@ func (ev *evaluator) eval(expr parser.Expr) (parser.Value, annotations.Annotatio T: ts, F: mat[i].Floats[0].F, }) + ev.currentSamples++ } else { - mat[i].Histograms = append(mat[i].Histograms, HPoint{ + point := HPoint{ T: ts, H: mat[i].Histograms[0].H, - }) + } + mat[i].Histograms = append(mat[i].Histograms, point) + ev.currentSamples += point.size() } - ev.currentSamples++ if ev.currentSamples > ev.maxSamples { ev.error(ErrTooManySamples(env)) } @@ -1857,9 +1864,14 @@ func (ev *evaluator) rangeEvalTimestampFunctionOverVectorSelector(vs *parser.Vec F: f, H: h, }) - + histSize := 0 + if h != nil { + histSize := h.Size() / 16 // 16 bytes per sample. + ev.currentSamples += histSize + } ev.currentSamples++ - ev.samplesStats.IncrementSamplesAtTimestamp(enh.Ts, 1) + + ev.samplesStats.IncrementSamplesAtTimestamp(enh.Ts, int64(1+histSize)) if ev.currentSamples > ev.maxSamples { ev.error(ErrTooManySamples(env)) } @@ -1981,10 +1993,10 @@ func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, annota } ss.Floats, ss.Histograms = ev.matrixIterSlice(it, mint, maxt, nil, nil) - totalLen := int64(len(ss.Floats)) + int64(len(ss.Histograms)) - ev.samplesStats.IncrementSamplesAtTimestamp(ev.startTimestamp, totalLen) + totalSize := int64(len(ss.Floats)) + int64(totalHPointSize(ss.Histograms)) + ev.samplesStats.IncrementSamplesAtTimestamp(ev.startTimestamp, totalSize) - if totalLen > 0 { + if totalSize > 0 { matrix = append(matrix, ss) } else { putFPointSlice(ss.Floats) @@ -2016,7 +2028,7 @@ func (ev *evaluator) matrixIterSlice( // (b) the number of samples is relatively small. // so a linear search will be as fast as a binary search. var drop int - for drop = 0; floats[drop].T < mint; drop++ { // nolint:revive + for drop = 0; floats[drop].T < mint; drop++ { } ev.currentSamples -= drop copy(floats, floats[drop:]) @@ -2038,15 +2050,15 @@ func (ev *evaluator) matrixIterSlice( // (b) the number of samples is relatively small. // so a linear search will be as fast as a binary search. var drop int - for drop = 0; histograms[drop].T < mint; drop++ { // nolint:revive + for drop = 0; histograms[drop].T < mint; drop++ { } - ev.currentSamples -= drop copy(histograms, histograms[drop:]) histograms = histograms[:len(histograms)-drop] + ev.currentSamples -= totalHPointSize(histograms) // Only append points with timestamps after the last timestamp we have. mintHistograms = histograms[len(histograms)-1].T + 1 } else { - ev.currentSamples -= len(histograms) + ev.currentSamples -= totalHPointSize(histograms) if histograms != nil { histograms = histograms[:0] } @@ -2075,11 +2087,12 @@ loop: if ev.currentSamples >= ev.maxSamples { ev.error(ErrTooManySamples(env)) } - ev.currentSamples++ + point := HPoint{T: t, H: h} if histograms == nil { histograms = getHPointSlice(16) } - histograms = append(histograms, HPoint{T: t, H: h}) + histograms = append(histograms, point) + ev.currentSamples += point.size() } case chunkenc.ValFloat: t, f := buf.At() @@ -2110,8 +2123,9 @@ loop: if histograms == nil { histograms = getHPointSlice(16) } - histograms = append(histograms, HPoint{T: t, H: h}) - ev.currentSamples++ + point := HPoint{T: t, H: h} + histograms = append(histograms, point) + ev.currentSamples += point.size() } case chunkenc.ValFloat: t, f := it.At() diff --git a/vendor/github.com/prometheus/prometheus/promql/functions.go b/vendor/github.com/prometheus/prometheus/promql/functions.go index 5fd54f6c76..06f6f8c71c 100644 --- a/vendor/github.com/prometheus/prometheus/promql/functions.go +++ b/vendor/github.com/prometheus/prometheus/promql/functions.go @@ -11,7 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive // Many unsued function arguments in this file by design. package promql import ( @@ -24,6 +23,7 @@ import ( "github.com/grafana/regexp" "github.com/prometheus/common/model" + "golang.org/x/exp/slices" "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" @@ -368,6 +368,68 @@ func funcSortDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHel return Vector(byValueSorter), nil } +// === sort_by_label(vector parser.ValueTypeVector, label parser.ValueTypeString...) (Vector, Annotations) === +func funcSortByLabel(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + // In case the labels are the same, NaN should sort to the bottom, so take + // ascending sort with NaN first and reverse it. + var anno annotations.Annotations + vals[0], anno = funcSort(vals, args, enh) + labels := stringSliceFromArgs(args[1:]) + slices.SortFunc(vals[0].(Vector), func(a, b Sample) int { + // Iterate over each given label + for _, label := range labels { + lv1 := a.Metric.Get(label) + lv2 := b.Metric.Get(label) + // 0 if a == b, -1 if a < b, and +1 if a > b. + switch strings.Compare(lv1, lv2) { + case -1: + return -1 + case +1: + return +1 + default: + continue + } + } + + return 0 + }) + + return vals[0].(Vector), anno +} + +// === sort_by_label_desc(vector parser.ValueTypeVector, label parser.ValueTypeString...) (Vector, Annotations) === +func funcSortByLabelDesc(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { + // In case the labels are the same, NaN should sort to the bottom, so take + // ascending sort with NaN first and reverse it. + var anno annotations.Annotations + vals[0], anno = funcSortDesc(vals, args, enh) + labels := stringSliceFromArgs(args[1:]) + slices.SortFunc(vals[0].(Vector), func(a, b Sample) int { + // Iterate over each given label + for _, label := range labels { + lv1 := a.Metric.Get(label) + lv2 := b.Metric.Get(label) + // If label values are the same, continue to the next label + if lv1 == lv2 { + continue + } + // 0 if a == b, -1 if a < b, and +1 if a > b. + switch strings.Compare(lv1, lv2) { + case -1: + return +1 + case +1: + return -1 + default: + continue + } + } + + return 0 + }) + + return vals[0].(Vector), anno +} + // === clamp(Vector parser.ValueTypeVector, min, max Scalar) (Vector, Annotations) === func funcClamp(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { vec := vals[0].(Vector) @@ -787,47 +849,47 @@ func funcTan(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) return simpleFunc(vals, enh, math.Tan), nil } -// == asin(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === asin(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcAsin(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Asin), nil } -// == acos(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === acos(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcAcos(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Acos), nil } -// == atan(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === atan(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcAtan(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Atan), nil } -// == sinh(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === sinh(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcSinh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Sinh), nil } -// == cosh(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === cosh(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcCosh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Cosh), nil } -// == tanh(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === tanh(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcTanh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Tanh), nil } -// == asinh(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === asinh(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcAsinh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Asinh), nil } -// == acosh(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === acosh(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcAcosh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Acosh), nil } -// == atanh(Vector parser.ValueTypeVector) (Vector, Annotations) === +// === atanh(Vector parser.ValueTypeVector) (Vector, Annotations) === func funcAtanh(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) (Vector, annotations.Annotations) { return simpleFunc(vals, enh, math.Atanh), nil } @@ -1114,7 +1176,7 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev var histogramSamples []Sample for _, sample := range inVec { - // We are only looking for conventional buckets here. Remember + // We are only looking for classic buckets here. Remember // the histograms for later treatment. if sample.H != nil { histogramSamples = append(histogramSamples, sample) @@ -1145,10 +1207,10 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev // Now deal with the histograms. for _, sample := range histogramSamples { // We have to reconstruct the exact same signature as above for - // a conventional histogram, just ignoring any le label. + // a classic histogram, just ignoring any le label. enh.lblBuf = sample.Metric.Bytes(enh.lblBuf) if mb, ok := enh.signatureToMetricWithBuckets[string(enh.lblBuf)]; ok && len(mb.buckets) > 0 { - // At this data point, we have conventional histogram + // At this data point, we have classic histogram // buckets and a native histogram with the same name and // labels. Do not evaluate anything. annos.Add(annotations.NewMixedClassicNativeHistogramsWarning(sample.Metric.Get(labels.MetricName), args[1].PositionRange())) @@ -1164,7 +1226,7 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev for _, mb := range enh.signatureToMetricWithBuckets { if len(mb.buckets) > 0 { - res, forcedMonotonicity := bucketQuantile(q, mb.buckets) + res, forcedMonotonicity, _ := bucketQuantile(q, mb.buckets) enh.Out = append(enh.Out, Sample{ Metric: mb.metric, F: res, @@ -1494,6 +1556,8 @@ var FunctionCalls = map[string]FunctionCall{ "sinh": funcSinh, "sort": funcSort, "sort_desc": funcSortDesc, + "sort_by_label": funcSortByLabel, + "sort_by_label_desc": funcSortByLabelDesc, "sqrt": funcSqrt, "stddev_over_time": funcStddevOverTime, "stdvar_over_time": funcStdvarOverTime, @@ -1638,3 +1702,11 @@ func stringFromArg(e parser.Expr) string { unwrapParenExpr(&tmp) // Optionally unwrap ParenExpr return tmp.(*parser.StringLiteral).Val } + +func stringSliceFromArgs(args parser.Expressions) []string { + tmp := make([]string, len(args)) + for i := 0; i < len(args); i++ { + tmp[i] = stringFromArg(args[i]) + } + return tmp +} diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/ast.go b/vendor/github.com/prometheus/prometheus/promql/parser/ast.go index 58136266fd..379352599d 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/ast.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/ast.go @@ -55,7 +55,6 @@ type Statement interface { Node // PromQLStmt ensures that no other type accidentally implements the interface - // nolint:unused PromQLStmt() } diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/functions.go b/vendor/github.com/prometheus/prometheus/promql/parser/functions.go index 45a30219e6..ee2e90c55e 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/functions.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/functions.go @@ -16,12 +16,16 @@ package parser // Function represents a function of the expression language and is // used by function nodes. type Function struct { - Name string - ArgTypes []ValueType - Variadic int - ReturnType ValueType + Name string + ArgTypes []ValueType + Variadic int + ReturnType ValueType + Experimental bool } +// EnableExperimentalFunctions controls whether experimentalFunctions are enabled. +var EnableExperimentalFunctions bool + // Functions is a list of all functions supported by PromQL, including their types. var Functions = map[string]*Function{ "abs": { @@ -343,6 +347,20 @@ var Functions = map[string]*Function{ ArgTypes: []ValueType{ValueTypeVector}, ReturnType: ValueTypeVector, }, + "sort_by_label": { + Name: "sort_by_label", + ArgTypes: []ValueType{ValueTypeVector, ValueTypeString}, + Variadic: -1, + ReturnType: ValueTypeVector, + Experimental: true, + }, + "sort_by_label_desc": { + Name: "sort_by_label_desc", + ArgTypes: []ValueType{ValueTypeVector, ValueTypeString}, + Variadic: -1, + ReturnType: ValueTypeVector, + Experimental: true, + }, "sqrt": { Name: "sqrt", ArgTypes: []ValueType{ValueTypeVector}, diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y index 676fd9fb5b..dce79f7693 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y +++ b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y @@ -22,7 +22,7 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/model/histogram" - "github.com/prometheus/prometheus/promql/parser/posrange" + "github.com/prometheus/prometheus/promql/parser/posrange" ) %} @@ -369,6 +369,9 @@ function_call : IDENTIFIER function_call_body if !exist{ yylex.(*parser).addParseErrf($1.PositionRange(),"unknown function with name %q", $1.Val) } + if fn != nil && fn.Experimental && !EnableExperimentalFunctions { + yylex.(*parser).addParseErrf($1.PositionRange(),"function %q is not enabled", $1.Val) + } $$ = &Call{ Func: fn, Args: $2.(Expressions), diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go index 77a403be35..4057d9163b 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/generated_parser.y.go @@ -230,7 +230,7 @@ const yyEofCode = 1 const yyErrCode = 2 const yyInitialStackSize = 16 -//line promql/parser/generated_parser.y:916 +//line promql/parser/generated_parser.y:919 //line yacctab:1 var yyExca = [...]int16{ @@ -1277,6 +1277,9 @@ yydefault: if !exist { yylex.(*parser).addParseErrf(yyDollar[1].item.PositionRange(), "unknown function with name %q", yyDollar[1].item.Val) } + if fn != nil && fn.Experimental && !EnableExperimentalFunctions { + yylex.(*parser).addParseErrf(yyDollar[1].item.PositionRange(), "function %q is not enabled", yyDollar[1].item.Val) + } yyVAL.node = &Call{ Func: fn, Args: yyDollar[2].node.(Expressions), @@ -1288,86 +1291,86 @@ yydefault: } case 61: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:384 +//line promql/parser/generated_parser.y:387 { yyVAL.node = yyDollar[2].node } case 62: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:386 +//line promql/parser/generated_parser.y:389 { yyVAL.node = Expressions{} } case 63: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:390 +//line promql/parser/generated_parser.y:393 { yyVAL.node = append(yyDollar[1].node.(Expressions), yyDollar[3].node.(Expr)) } case 64: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:392 +//line promql/parser/generated_parser.y:395 { yyVAL.node = Expressions{yyDollar[1].node.(Expr)} } case 65: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:394 +//line promql/parser/generated_parser.y:397 { yylex.(*parser).addParseErrf(yyDollar[2].item.PositionRange(), "trailing commas not allowed in function call args") yyVAL.node = yyDollar[1].node } case 66: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:405 +//line promql/parser/generated_parser.y:408 { yyVAL.node = &ParenExpr{Expr: yyDollar[2].node.(Expr), PosRange: mergeRanges(&yyDollar[1].item, &yyDollar[3].item)} } case 67: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:413 +//line promql/parser/generated_parser.y:416 { yylex.(*parser).addOffset(yyDollar[1].node, yyDollar[3].duration) yyVAL.node = yyDollar[1].node } case 68: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:418 +//line promql/parser/generated_parser.y:421 { yylex.(*parser).addOffset(yyDollar[1].node, -yyDollar[4].duration) yyVAL.node = yyDollar[1].node } case 69: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:423 +//line promql/parser/generated_parser.y:426 { yylex.(*parser).unexpected("offset", "duration") yyVAL.node = yyDollar[1].node } case 70: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:430 +//line promql/parser/generated_parser.y:433 { yylex.(*parser).setTimestamp(yyDollar[1].node, yyDollar[3].float) yyVAL.node = yyDollar[1].node } case 71: yyDollar = yyS[yypt-5 : yypt+1] -//line promql/parser/generated_parser.y:435 +//line promql/parser/generated_parser.y:438 { yylex.(*parser).setAtModifierPreprocessor(yyDollar[1].node, yyDollar[3].item) yyVAL.node = yyDollar[1].node } case 72: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:440 +//line promql/parser/generated_parser.y:443 { yylex.(*parser).unexpected("@", "timestamp") yyVAL.node = yyDollar[1].node } case 75: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:450 +//line promql/parser/generated_parser.y:453 { var errMsg string vs, ok := yyDollar[1].node.(*VectorSelector) @@ -1392,7 +1395,7 @@ yydefault: } case 76: yyDollar = yyS[yypt-6 : yypt+1] -//line promql/parser/generated_parser.y:475 +//line promql/parser/generated_parser.y:478 { yyVAL.node = &SubqueryExpr{ Expr: yyDollar[1].node.(Expr), @@ -1404,35 +1407,35 @@ yydefault: } case 77: yyDollar = yyS[yypt-6 : yypt+1] -//line promql/parser/generated_parser.y:485 +//line promql/parser/generated_parser.y:488 { yylex.(*parser).unexpected("subquery selector", "\"]\"") yyVAL.node = yyDollar[1].node } case 78: yyDollar = yyS[yypt-5 : yypt+1] -//line promql/parser/generated_parser.y:487 +//line promql/parser/generated_parser.y:490 { yylex.(*parser).unexpected("subquery selector", "duration or \"]\"") yyVAL.node = yyDollar[1].node } case 79: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:489 +//line promql/parser/generated_parser.y:492 { yylex.(*parser).unexpected("subquery or range", "\":\" or \"]\"") yyVAL.node = yyDollar[1].node } case 80: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:491 +//line promql/parser/generated_parser.y:494 { yylex.(*parser).unexpected("subquery selector", "duration") yyVAL.node = yyDollar[1].node } case 81: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:501 +//line promql/parser/generated_parser.y:504 { if nl, ok := yyDollar[2].node.(*NumberLiteral); ok { if yyDollar[1].item.Typ == SUB { @@ -1446,7 +1449,7 @@ yydefault: } case 82: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:519 +//line promql/parser/generated_parser.y:522 { vs := yyDollar[2].node.(*VectorSelector) vs.PosRange = mergeRanges(&yyDollar[1].item, vs) @@ -1456,7 +1459,7 @@ yydefault: } case 83: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:527 +//line promql/parser/generated_parser.y:530 { vs := &VectorSelector{ Name: yyDollar[1].item.Val, @@ -1468,7 +1471,7 @@ yydefault: } case 84: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:537 +//line promql/parser/generated_parser.y:540 { vs := yyDollar[1].node.(*VectorSelector) yylex.(*parser).assembleVectorSelector(vs) @@ -1476,7 +1479,7 @@ yydefault: } case 85: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:545 +//line promql/parser/generated_parser.y:548 { yyVAL.node = &VectorSelector{ LabelMatchers: yyDollar[2].matchers, @@ -1485,7 +1488,7 @@ yydefault: } case 86: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:552 +//line promql/parser/generated_parser.y:555 { yyVAL.node = &VectorSelector{ LabelMatchers: yyDollar[2].matchers, @@ -1494,7 +1497,7 @@ yydefault: } case 87: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:559 +//line promql/parser/generated_parser.y:562 { yyVAL.node = &VectorSelector{ LabelMatchers: []*labels.Matcher{}, @@ -1503,7 +1506,7 @@ yydefault: } case 88: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:568 +//line promql/parser/generated_parser.y:571 { if yyDollar[1].matchers != nil { yyVAL.matchers = append(yyDollar[1].matchers, yyDollar[3].matcher) @@ -1513,47 +1516,47 @@ yydefault: } case 89: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:576 +//line promql/parser/generated_parser.y:579 { yyVAL.matchers = []*labels.Matcher{yyDollar[1].matcher} } case 90: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:578 +//line promql/parser/generated_parser.y:581 { yylex.(*parser).unexpected("label matching", "\",\" or \"}\"") yyVAL.matchers = yyDollar[1].matchers } case 91: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:582 +//line promql/parser/generated_parser.y:585 { yyVAL.matcher = yylex.(*parser).newLabelMatcher(yyDollar[1].item, yyDollar[2].item, yyDollar[3].item) } case 92: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:584 +//line promql/parser/generated_parser.y:587 { yylex.(*parser).unexpected("label matching", "string") yyVAL.matcher = nil } case 93: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:586 +//line promql/parser/generated_parser.y:589 { yylex.(*parser).unexpected("label matching", "label matching operator") yyVAL.matcher = nil } case 94: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:588 +//line promql/parser/generated_parser.y:591 { yylex.(*parser).unexpected("label matching", "identifier or \"}\"") yyVAL.matcher = nil } case 95: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:596 +//line promql/parser/generated_parser.y:599 { b := labels.NewBuilder(yyDollar[2].labels) b.Set(labels.MetricName, yyDollar[1].item.Val) @@ -1561,83 +1564,83 @@ yydefault: } case 96: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:598 +//line promql/parser/generated_parser.y:601 { yyVAL.labels = yyDollar[1].labels } case 119: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:605 +//line promql/parser/generated_parser.y:608 { yyVAL.labels = labels.New(yyDollar[2].lblList...) } case 120: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:607 +//line promql/parser/generated_parser.y:610 { yyVAL.labels = labels.New(yyDollar[2].lblList...) } case 121: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:609 +//line promql/parser/generated_parser.y:612 { yyVAL.labels = labels.New() } case 122: yyDollar = yyS[yypt-0 : yypt+1] -//line promql/parser/generated_parser.y:611 +//line promql/parser/generated_parser.y:614 { yyVAL.labels = labels.New() } case 123: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:615 +//line promql/parser/generated_parser.y:618 { yyVAL.lblList = append(yyDollar[1].lblList, yyDollar[3].label) } case 124: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:617 +//line promql/parser/generated_parser.y:620 { yyVAL.lblList = []labels.Label{yyDollar[1].label} } case 125: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:619 +//line promql/parser/generated_parser.y:622 { yylex.(*parser).unexpected("label set", "\",\" or \"}\"") yyVAL.lblList = yyDollar[1].lblList } case 126: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:624 +//line promql/parser/generated_parser.y:627 { yyVAL.label = labels.Label{Name: yyDollar[1].item.Val, Value: yylex.(*parser).unquoteString(yyDollar[3].item.Val)} } case 127: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:626 +//line promql/parser/generated_parser.y:629 { yylex.(*parser).unexpected("label set", "string") yyVAL.label = labels.Label{} } case 128: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:628 +//line promql/parser/generated_parser.y:631 { yylex.(*parser).unexpected("label set", "\"=\"") yyVAL.label = labels.Label{} } case 129: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:630 +//line promql/parser/generated_parser.y:633 { yylex.(*parser).unexpected("label set", "identifier or \"}\"") yyVAL.label = labels.Label{} } case 130: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:641 +//line promql/parser/generated_parser.y:644 { yylex.(*parser).generatedParserResult = &seriesDescription{ labels: yyDollar[1].labels, @@ -1646,38 +1649,38 @@ yydefault: } case 131: yyDollar = yyS[yypt-0 : yypt+1] -//line promql/parser/generated_parser.y:650 +//line promql/parser/generated_parser.y:653 { yyVAL.series = []SequenceValue{} } case 132: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:652 +//line promql/parser/generated_parser.y:655 { yyVAL.series = append(yyDollar[1].series, yyDollar[3].series...) } case 133: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:654 +//line promql/parser/generated_parser.y:657 { yyVAL.series = yyDollar[1].series } case 134: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:656 +//line promql/parser/generated_parser.y:659 { yylex.(*parser).unexpected("series values", "") yyVAL.series = nil } case 135: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:660 +//line promql/parser/generated_parser.y:663 { yyVAL.series = []SequenceValue{{Omitted: true}} } case 136: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:662 +//line promql/parser/generated_parser.y:665 { yyVAL.series = []SequenceValue{} for i := uint64(0); i < yyDollar[3].uint; i++ { @@ -1686,13 +1689,13 @@ yydefault: } case 137: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:669 +//line promql/parser/generated_parser.y:672 { yyVAL.series = []SequenceValue{{Value: yyDollar[1].float}} } case 138: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:671 +//line promql/parser/generated_parser.y:674 { yyVAL.series = []SequenceValue{} // Add an additional value for time 0, which we ignore in tests. @@ -1702,7 +1705,7 @@ yydefault: } case 139: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:679 +//line promql/parser/generated_parser.y:682 { yyVAL.series = []SequenceValue{} // Add an additional value for time 0, which we ignore in tests. @@ -1713,13 +1716,13 @@ yydefault: } case 140: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:689 +//line promql/parser/generated_parser.y:692 { yyVAL.series = []SequenceValue{{Histogram: yyDollar[1].histogram}} } case 141: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:693 +//line promql/parser/generated_parser.y:696 { yyVAL.series = []SequenceValue{} // Add an additional value for time 0, which we ignore in tests. @@ -1730,7 +1733,7 @@ yydefault: } case 142: yyDollar = yyS[yypt-5 : yypt+1] -//line promql/parser/generated_parser.y:702 +//line promql/parser/generated_parser.y:705 { val, err := yylex.(*parser).histogramsIncreaseSeries(yyDollar[1].histogram, yyDollar[3].histogram, yyDollar[5].uint) if err != nil { @@ -1740,7 +1743,7 @@ yydefault: } case 143: yyDollar = yyS[yypt-5 : yypt+1] -//line promql/parser/generated_parser.y:710 +//line promql/parser/generated_parser.y:713 { val, err := yylex.(*parser).histogramsDecreaseSeries(yyDollar[1].histogram, yyDollar[3].histogram, yyDollar[5].uint) if err != nil { @@ -1750,7 +1753,7 @@ yydefault: } case 144: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:720 +//line promql/parser/generated_parser.y:723 { if yyDollar[1].item.Val != "stale" { yylex.(*parser).unexpected("series values", "number or \"stale\"") @@ -1759,138 +1762,138 @@ yydefault: } case 147: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:732 +//line promql/parser/generated_parser.y:735 { yyVAL.histogram = yylex.(*parser).buildHistogramFromMap(&yyDollar[2].descriptors) } case 148: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:736 +//line promql/parser/generated_parser.y:739 { yyVAL.histogram = yylex.(*parser).buildHistogramFromMap(&yyDollar[2].descriptors) } case 149: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:740 +//line promql/parser/generated_parser.y:743 { m := yylex.(*parser).newMap() yyVAL.histogram = yylex.(*parser).buildHistogramFromMap(&m) } case 150: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:745 +//line promql/parser/generated_parser.y:748 { m := yylex.(*parser).newMap() yyVAL.histogram = yylex.(*parser).buildHistogramFromMap(&m) } case 151: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:753 +//line promql/parser/generated_parser.y:756 { yyVAL.descriptors = *(yylex.(*parser).mergeMaps(&yyDollar[1].descriptors, &yyDollar[3].descriptors)) } case 152: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:757 +//line promql/parser/generated_parser.y:760 { yyVAL.descriptors = yyDollar[1].descriptors } case 153: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:760 +//line promql/parser/generated_parser.y:763 { yylex.(*parser).unexpected("histogram description", "histogram description key, e.g. buckets:[5 10 7]") } case 154: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:767 +//line promql/parser/generated_parser.y:770 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["schema"] = yyDollar[3].int } case 155: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:772 +//line promql/parser/generated_parser.y:775 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["sum"] = yyDollar[3].float } case 156: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:777 +//line promql/parser/generated_parser.y:780 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["count"] = yyDollar[3].float } case 157: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:782 +//line promql/parser/generated_parser.y:785 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["z_bucket"] = yyDollar[3].float } case 158: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:787 +//line promql/parser/generated_parser.y:790 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["z_bucket_w"] = yyDollar[3].float } case 159: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:792 +//line promql/parser/generated_parser.y:795 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["buckets"] = yyDollar[3].bucket_set } case 160: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:797 +//line promql/parser/generated_parser.y:800 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["offset"] = yyDollar[3].int } case 161: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:802 +//line promql/parser/generated_parser.y:805 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["n_buckets"] = yyDollar[3].bucket_set } case 162: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:807 +//line promql/parser/generated_parser.y:810 { yyVAL.descriptors = yylex.(*parser).newMap() yyVAL.descriptors["n_offset"] = yyDollar[3].int } case 163: yyDollar = yyS[yypt-4 : yypt+1] -//line promql/parser/generated_parser.y:814 +//line promql/parser/generated_parser.y:817 { yyVAL.bucket_set = yyDollar[2].bucket_set } case 164: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:818 +//line promql/parser/generated_parser.y:821 { yyVAL.bucket_set = yyDollar[2].bucket_set } case 165: yyDollar = yyS[yypt-3 : yypt+1] -//line promql/parser/generated_parser.y:824 +//line promql/parser/generated_parser.y:827 { yyVAL.bucket_set = append(yyDollar[1].bucket_set, yyDollar[3].float) } case 166: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:828 +//line promql/parser/generated_parser.y:831 { yyVAL.bucket_set = []float64{yyDollar[1].float} } case 213: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:853 +//line promql/parser/generated_parser.y:856 { yyVAL.node = &NumberLiteral{ Val: yylex.(*parser).number(yyDollar[1].item.Val), @@ -1899,25 +1902,25 @@ yydefault: } case 214: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:861 +//line promql/parser/generated_parser.y:864 { yyVAL.float = yylex.(*parser).number(yyDollar[1].item.Val) } case 215: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:863 +//line promql/parser/generated_parser.y:866 { yyVAL.float = yyDollar[2].float } case 216: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:864 +//line promql/parser/generated_parser.y:867 { yyVAL.float = -yyDollar[2].float } case 219: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:870 +//line promql/parser/generated_parser.y:873 { var err error yyVAL.uint, err = strconv.ParseUint(yyDollar[1].item.Val, 10, 64) @@ -1927,19 +1930,19 @@ yydefault: } case 220: yyDollar = yyS[yypt-2 : yypt+1] -//line promql/parser/generated_parser.y:879 +//line promql/parser/generated_parser.y:882 { yyVAL.int = -int64(yyDollar[2].uint) } case 221: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:880 +//line promql/parser/generated_parser.y:883 { yyVAL.int = int64(yyDollar[1].uint) } case 222: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:884 +//line promql/parser/generated_parser.y:887 { var err error yyVAL.duration, err = parseDuration(yyDollar[1].item.Val) @@ -1949,7 +1952,7 @@ yydefault: } case 223: yyDollar = yyS[yypt-1 : yypt+1] -//line promql/parser/generated_parser.y:895 +//line promql/parser/generated_parser.y:898 { yyVAL.node = &StringLiteral{ Val: yylex.(*parser).unquoteString(yyDollar[1].item.Val), @@ -1958,13 +1961,13 @@ yydefault: } case 224: yyDollar = yyS[yypt-0 : yypt+1] -//line promql/parser/generated_parser.y:908 +//line promql/parser/generated_parser.y:911 { yyVAL.duration = 0 } case 226: yyDollar = yyS[yypt-0 : yypt+1] -//line promql/parser/generated_parser.y:912 +//line promql/parser/generated_parser.y:915 { yyVAL.strings = nil } diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/lex.go b/vendor/github.com/prometheus/prometheus/promql/parser/lex.go index c8bfcc2e1e..641bedcfcc 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/lex.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/lex.go @@ -11,7 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive // Many legitimately empty blocks in this file. package parser import ( @@ -60,11 +59,11 @@ func (i Item) Pretty(int) string { return i.String() } func (i ItemType) IsOperator() bool { return i > operatorsStart && i < operatorsEnd } // IsAggregator returns true if the Item belongs to the aggregator functions. -// Returns false otherwise +// Returns false otherwise. func (i ItemType) IsAggregator() bool { return i > aggregatorsStart && i < aggregatorsEnd } // IsAggregatorWithParam returns true if the Item is an aggregator that takes a parameter. -// Returns false otherwise +// Returns false otherwise. func (i ItemType) IsAggregatorWithParam() bool { return i == TOPK || i == BOTTOMK || i == COUNT_VALUES || i == QUANTILE } @@ -567,9 +566,8 @@ Loop: if l.peek() == ':' { l.emit(desc) return lexHistogram - } else { - l.errorf("missing `:` for histogram descriptor") } + l.errorf("missing `:` for histogram descriptor") } else { l.errorf("bad histogram descriptor found: %q", word) } diff --git a/vendor/github.com/prometheus/prometheus/promql/parser/parse.go b/vendor/github.com/prometheus/prometheus/promql/parser/parse.go index 34217697a6..122286c551 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parser/parse.go +++ b/vendor/github.com/prometheus/prometheus/promql/parser/parse.go @@ -72,8 +72,7 @@ func WithFunctions(functions map[string]*Function) Opt { } // NewParser returns a new parser. -// nolint:revive -func NewParser(input string, opts ...Opt) *parser { +func NewParser(input string, opts ...Opt) *parser { //nolint:revive // unexported-return. p := parserPool.Get().(*parser) p.functions = Functions @@ -172,7 +171,7 @@ func ParseExpr(input string) (expr Expr, err error) { return p.ParseExpr() } -// ParseMetric parses the input into a metric +// ParseMetric parses the input into a metric. func ParseMetric(input string) (m labels.Labels, err error) { p := NewParser(input) defer p.Close() @@ -660,9 +659,9 @@ func (p *parser) checkAST(node Node) (typ ValueType) { // This is made a function instead of a variable, so it is lazily evaluated on demand. opRange := func() (r posrange.PositionRange) { // Remove whitespace at the beginning and end of the range. - for r.Start = n.LHS.PositionRange().End; isSpace(rune(p.lex.input[r.Start])); r.Start++ { // nolint:revive + for r.Start = n.LHS.PositionRange().End; isSpace(rune(p.lex.input[r.Start])); r.Start++ { } - for r.End = n.RHS.PositionRange().Start - 1; isSpace(rune(p.lex.input[r.End])); r.End-- { // nolint:revive + for r.End = n.RHS.PositionRange().Start - 1; isSpace(rune(p.lex.input[r.End])); r.End-- { } return } diff --git a/vendor/github.com/prometheus/prometheus/promql/quantile.go b/vendor/github.com/prometheus/prometheus/promql/quantile.go index f289448682..f62519f5b9 100644 --- a/vendor/github.com/prometheus/prometheus/promql/quantile.go +++ b/vendor/github.com/prometheus/prometheus/promql/quantile.go @@ -23,6 +23,25 @@ import ( "github.com/prometheus/prometheus/model/labels" ) +// smallDeltaTolerance is the threshold for relative deltas between classic +// histogram buckets that will be ignored by the histogram_quantile function +// because they are most likely artifacts of floating point precision issues. +// Testing on 2 sets of real data with bugs arising from small deltas, +// the safe ranges were from: +// - 1e-05 to 1e-15 +// - 1e-06 to 1e-15 +// Anything to the left of that would cause non-query-sharded data to have +// small deltas ignored (unnecessary and we should avoid this), and anything +// to the right of that would cause query-sharded data to not have its small +// deltas ignored (so the problem won't be fixed). +// For context, query sharding triggers these float precision errors in Mimir. +// To illustrate, with a relative deviation of 1e-12, we need to have 1e12 +// observations in the bucket so that the change of one observation is small +// enough to get ignored. With the usual observation rate even of very busy +// services, this will hardly be reached in timeframes that matters for +// monitoring. +const smallDeltaTolerance = 1e-12 + // Helpers to calculate quantiles. // excludedLabels are the labels to exclude from signature calculation for @@ -72,16 +91,19 @@ type metricWithBuckets struct { // // If q>1, +Inf is returned. // -// We also return a bool to indicate if monotonicity needed to be forced. -func bucketQuantile(q float64, buckets buckets) (float64, bool) { +// We also return a bool to indicate if monotonicity needed to be forced, +// and another bool to indicate if small differences between buckets (that +// are likely artifacts of floating point precision issues) have been +// ignored. +func bucketQuantile(q float64, buckets buckets) (float64, bool, bool) { if math.IsNaN(q) { - return math.NaN(), false + return math.NaN(), false, false } if q < 0 { - return math.Inf(-1), false + return math.Inf(-1), false, false } if q > 1 { - return math.Inf(+1), false + return math.Inf(+1), false, false } slices.SortFunc(buckets, func(a, b bucket) int { // We don't expect the bucket boundary to be a NaN. @@ -94,27 +116,27 @@ func bucketQuantile(q float64, buckets buckets) (float64, bool) { return 0 }) if !math.IsInf(buckets[len(buckets)-1].upperBound, +1) { - return math.NaN(), false + return math.NaN(), false, false } buckets = coalesceBuckets(buckets) - forcedMonotonic := ensureMonotonic(buckets) + forcedMonotonic, fixedPrecision := ensureMonotonicAndIgnoreSmallDeltas(buckets, smallDeltaTolerance) if len(buckets) < 2 { - return math.NaN(), false + return math.NaN(), false, false } observations := buckets[len(buckets)-1].count if observations == 0 { - return math.NaN(), false + return math.NaN(), false, false } rank := q * observations b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank }) if b == len(buckets)-1 { - return buckets[len(buckets)-2].upperBound, forcedMonotonic + return buckets[len(buckets)-2].upperBound, forcedMonotonic, fixedPrecision } if b == 0 && buckets[0].upperBound <= 0 { - return buckets[0].upperBound, forcedMonotonic + return buckets[0].upperBound, forcedMonotonic, fixedPrecision } var ( bucketStart float64 @@ -126,7 +148,7 @@ func bucketQuantile(q float64, buckets buckets) (float64, bool) { count -= buckets[b-1].count rank -= buckets[b-1].count } - return bucketStart + (bucketEnd-bucketStart)*(rank/count), forcedMonotonic + return bucketStart + (bucketEnd-bucketStart)*(rank/count), forcedMonotonic, fixedPrecision } // histogramQuantile calculates the quantile 'q' based on the given histogram. @@ -348,6 +370,7 @@ func coalesceBuckets(buckets buckets) buckets { // - Ingestion via the remote write receiver that Prometheus implements. // - Optimisation of query execution where precision is sacrificed for other // benefits, not by Prometheus but by systems built on top of it. +// - Circumstances where floating point precision errors accumulate. // // Monotonicity is usually guaranteed because if a bucket with upper bound // u1 has count c1, then any bucket with a higher upper bound u > u1 must @@ -357,22 +380,42 @@ func coalesceBuckets(buckets buckets) buckets { // bucket with the φ-quantile count, so breaking the monotonicity // guarantee causes bucketQuantile() to return undefined (nonsense) results. // -// As a somewhat hacky solution, we calculate the "envelope" of the histogram -// buckets, essentially removing any decreases in the count between successive -// buckets. We return a bool to indicate if this monotonicity was forced or not. -func ensureMonotonic(buckets buckets) bool { - forced := false - max := buckets[0].count +// As a somewhat hacky solution, we first silently ignore any numerically +// insignificant (relative delta below the requested tolerance and likely to +// be from floating point precision errors) differences between successive +// buckets regardless of the direction. Then we calculate the "envelope" of +// the histogram buckets, essentially removing any decreases in the count +// between successive buckets. +// +// We return a bool to indicate if this monotonicity was forced or not, and +// another bool to indicate if small deltas were ignored or not. +func ensureMonotonicAndIgnoreSmallDeltas(buckets buckets, tolerance float64) (bool, bool) { + var forcedMonotonic, fixedPrecision bool + prev := buckets[0].count for i := 1; i < len(buckets); i++ { - switch { - case buckets[i].count > max: - max = buckets[i].count - case buckets[i].count < max: - buckets[i].count = max - forced = true + curr := buckets[i].count // Assumed always positive. + if curr == prev { + // No correction needed if the counts are identical between buckets. + continue + } + if almostEqual(prev, curr, tolerance) { + // Silently correct numerically insignificant differences from floating + // point precision errors, regardless of direction. + // Do not update the 'prev' value as we are ignoring the difference. + buckets[i].count = prev + fixedPrecision = true + continue + } + if curr < prev { + // Force monotonicity by removing any decreases regardless of magnitude. + // Do not update the 'prev' value as we are ignoring the decrease. + buckets[i].count = prev + forcedMonotonic = true + continue } + prev = curr } - return forced + return forcedMonotonic, fixedPrecision } // quantile calculates the given quantile of a vector of samples. diff --git a/vendor/github.com/prometheus/prometheus/promql/test.go b/vendor/github.com/prometheus/prometheus/promql/test.go index f6a31ee431..589b1e5b6b 100644 --- a/vendor/github.com/prometheus/prometheus/promql/test.go +++ b/vendor/github.com/prometheus/prometheus/promql/test.go @@ -49,7 +49,7 @@ var ( ) const ( - epsilon = 0.000001 // Relative error allowed for sample values. + defaultEpsilon = 0.000001 // Relative error allowed for sample values. ) var testStartTime = time.Unix(0, 0).UTC() @@ -73,6 +73,9 @@ func LoadedStorage(t testutil.T, input string) *teststorage.TestStorage { // RunBuiltinTests runs an acceptance test suite against the provided engine. func RunBuiltinTests(t *testing.T, engine engineQuerier) { + t.Cleanup(func() { parser.EnableExperimentalFunctions = false }) + parser.EnableExperimentalFunctions = true + files, err := fs.Glob(testsFs, "*/*.test") require.NoError(t, err) @@ -440,7 +443,7 @@ func (ev *evalCmd) compareResult(result parser.Value) error { if (expH == nil) != (v.H == nil) || (expH != nil && !expH.Equals(v.H)) { return fmt.Errorf("expected %v for %s but got %s", HistogramTestExpression(expH), v.Metric, HistogramTestExpression(v.H)) } - if !almostEqual(exp0.Value, v.F) { + if !almostEqual(exp0.Value, v.F, defaultEpsilon) { return fmt.Errorf("expected %v for %s but got %v", exp0.Value, v.Metric, v.F) } @@ -464,7 +467,7 @@ func (ev *evalCmd) compareResult(result parser.Value) error { if exp0.Histogram != nil { return fmt.Errorf("expected Histogram %v but got scalar %s", exp0.Histogram.TestExpression(), val.String()) } - if !almostEqual(exp0.Value, val.V) { + if !almostEqual(exp0.Value, val.V, defaultEpsilon) { return fmt.Errorf("expected Scalar %v but got %v", val.V, exp0.Value) } @@ -663,9 +666,9 @@ func (t *test) clear() { t.context, t.cancelCtx = context.WithCancel(context.Background()) } -// samplesAlmostEqual returns true if the two sample lines only differ by a -// small relative error in their sample value. -func almostEqual(a, b float64) bool { +// almostEqual returns true if a and b differ by less than their sum +// multiplied by epsilon. +func almostEqual(a, b, epsilon float64) bool { // NaN has no equality but for testing we still want to know whether both values // are NaN. if math.IsNaN(a) && math.IsNaN(b) { @@ -677,12 +680,13 @@ func almostEqual(a, b float64) bool { return true } + absSum := math.Abs(a) + math.Abs(b) diff := math.Abs(a - b) - if a == 0 || b == 0 || diff < minNormal { + if a == 0 || b == 0 || absSum < minNormal { return diff < epsilon*minNormal } - return diff/(math.Abs(a)+math.Abs(b)) < epsilon + return diff/math.Min(absSum, math.MaxFloat64) < epsilon } func parseNumber(s string) (float64, error) { diff --git a/vendor/github.com/prometheus/prometheus/promql/testdata/functions.test b/vendor/github.com/prometheus/prometheus/promql/testdata/functions.test index 02e6a32190..b5263a96fc 100644 --- a/vendor/github.com/prometheus/prometheus/promql/testdata/functions.test +++ b/vendor/github.com/prometheus/prometheus/promql/testdata/functions.test @@ -469,6 +469,116 @@ eval_ordered instant at 50m sort_desc(http_requests) http_requests{group="production", instance="0", job="api-server"} 100 http_requests{group="canary", instance="2", job="api-server"} NaN +# Tests for sort_by_label/sort_by_label_desc. +clear +load 5m + http_requests{job="api-server", instance="0", group="production"} 0+10x10 + http_requests{job="api-server", instance="1", group="production"} 0+20x10 + http_requests{job="api-server", instance="0", group="canary"} 0+30x10 + http_requests{job="api-server", instance="1", group="canary"} 0+40x10 + http_requests{job="api-server", instance="2", group="canary"} NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN + http_requests{job="app-server", instance="0", group="production"} 0+50x10 + http_requests{job="app-server", instance="1", group="production"} 0+60x10 + http_requests{job="app-server", instance="0", group="canary"} 0+70x10 + http_requests{job="app-server", instance="1", group="canary"} 0+80x10 + http_requests{job="api-server", instance="2", group="production"} 0+10x10 + +eval_ordered instant at 50m sort_by_label(http_requests, "instance") + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="production", instance="2", job="api-server"} 100 + http_requests{group="canary", instance="2", job="api-server"} NaN + +eval_ordered instant at 50m sort_by_label(http_requests, "instance", "group") + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="canary", instance="2", job="api-server"} NaN + http_requests{group="production", instance="2", job="api-server"} 100 + +eval_ordered instant at 50m sort_by_label(http_requests, "instance", "group") + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="canary", instance="2", job="api-server"} NaN + http_requests{group="production", instance="2", job="api-server"} 100 + +eval_ordered instant at 50m sort_by_label(http_requests, "group", "instance", "job") + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="2", job="api-server"} NaN + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="production", instance="2", job="api-server"} 100 + +eval_ordered instant at 50m sort_by_label(http_requests, "job", "instance", "group") + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="canary", instance="2", job="api-server"} NaN + http_requests{group="production", instance="2", job="api-server"} 100 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="production", instance="1", job="app-server"} 600 + +eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance") + http_requests{group="production", instance="2", job="api-server"} 100 + http_requests{group="canary", instance="2", job="api-server"} NaN + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="canary", instance="0", job="api-server"} 300 + http_requests{group="production", instance="0", job="api-server"} 100 + +eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance", "group") + http_requests{group="production", instance="2", job="api-server"} 100 + http_requests{group="canary", instance="2", job="api-server"} NaN + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="canary", instance="0", job="api-server"} 300 + +eval_ordered instant at 50m sort_by_label_desc(http_requests, "instance", "group", "job") + http_requests{group="production", instance="2", job="api-server"} 100 + http_requests{group="canary", instance="2", job="api-server"} NaN + http_requests{group="production", instance="1", job="app-server"} 600 + http_requests{group="production", instance="1", job="api-server"} 200 + http_requests{group="canary", instance="1", job="app-server"} 800 + http_requests{group="canary", instance="1", job="api-server"} 400 + http_requests{group="production", instance="0", job="app-server"} 500 + http_requests{group="production", instance="0", job="api-server"} 100 + http_requests{group="canary", instance="0", job="app-server"} 700 + http_requests{group="canary", instance="0", job="api-server"} 300 + # Tests for holt_winters clear diff --git a/vendor/github.com/prometheus/prometheus/promql/value.go b/vendor/github.com/prometheus/prometheus/promql/value.go index 68e37f37ee..28cf3fe31c 100644 --- a/vendor/github.com/prometheus/prometheus/promql/value.go +++ b/vendor/github.com/prometheus/prometheus/promql/value.go @@ -168,6 +168,23 @@ func (p HPoint) MarshalJSON() ([]byte, error) { return json.Marshal([...]interface{}{float64(p.T) / 1000, h}) } +// size returns the size of the HPoint compared to the size of an FPoint. +// The total size is calculated considering the histogram timestamp (p.T - 8 bytes), +// and then a number of bytes in the histogram. +// This sum is divided by 16, as samples are 16 bytes. +func (p HPoint) size() int { + return (p.H.Size() + 8) / 16 +} + +// totalHPointSize returns the total number of samples in the given slice of HPoints. +func totalHPointSize(histograms []HPoint) int { + var total int + for _, h := range histograms { + total += h.size() + } + return total +} + // Sample is a single sample belonging to a metric. It represents either a float // sample or a histogram sample. If H is nil, it is a float sample. Otherwise, // it is a histogram sample. @@ -226,6 +243,21 @@ func (vec Vector) String() string { return strings.Join(entries, "\n") } +// TotalSamples returns the total number of samples in the series within a vector. +// Float samples have a weight of 1 in this number, while histogram samples have a higher +// weight according to their size compared with the size of a float sample. +// See HPoint.size for details. +func (vec Vector) TotalSamples() int { + numSamples := 0 + for _, sample := range vec { + numSamples++ + if sample.H != nil { + numSamples += sample.H.Size() / 16 + } + } + return numSamples +} + // ContainsSameLabelset checks if a vector has samples with the same labelset // Such a behavior is semantically undefined // https://github.com/prometheus/prometheus/issues/4562 @@ -264,10 +296,13 @@ func (m Matrix) String() string { } // TotalSamples returns the total number of samples in the series within a matrix. +// Float samples have a weight of 1 in this number, while histogram samples have a higher +// weight according to their size compared with the size of a float sample. +// See HPoint.size for details. func (m Matrix) TotalSamples() int { numSamples := 0 for _, series := range m { - numSamples += len(series.Floats) + len(series.Histograms) + numSamples += len(series.Floats) + totalHPointSize(series.Histograms) } return numSamples } diff --git a/vendor/github.com/prometheus/prometheus/rules/alerting.go b/vendor/github.com/prometheus/prometheus/rules/alerting.go index 228809486e..72a3c7913e 100644 --- a/vendor/github.com/prometheus/prometheus/rules/alerting.go +++ b/vendor/github.com/prometheus/prometheus/rules/alerting.go @@ -472,7 +472,7 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, } // State returns the maximum state of alert instances for this rule. -// StateFiring > StatePending > StateInactive +// StateFiring > StatePending > StateInactive. func (r *AlertingRule) State() AlertState { r.activeMtx.Lock() defer r.activeMtx.Unlock() diff --git a/vendor/github.com/prometheus/prometheus/rules/group.go b/vendor/github.com/prometheus/prometheus/rules/group.go new file mode 100644 index 0000000000..55673452e5 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/rules/group.go @@ -0,0 +1,868 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rules + +import ( + "context" + "errors" + "math" + "strings" + "sync" + "time" + + "golang.org/x/exp/slices" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/timestamp" + "github.com/prometheus/prometheus/model/value" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" +) + +// Group is a set of rules that have a logical relation. +type Group struct { + name string + file string + interval time.Duration + limit int + rules []Rule + seriesInPreviousEval []map[string]labels.Labels // One per Rule. + staleSeries []labels.Labels + opts *ManagerOptions + mtx sync.Mutex + evaluationTime time.Duration + lastEvaluation time.Time // Wall-clock time of most recent evaluation. + lastEvalTimestamp time.Time // Time slot used for most recent evaluation. + + shouldRestore bool + + markStale bool + done chan struct{} + terminated chan struct{} + managerDone chan struct{} + + logger log.Logger + + metrics *Metrics + + // Rule group evaluation iteration function, + // defaults to DefaultEvalIterationFunc. + evalIterationFunc GroupEvalIterationFunc +} + +// GroupEvalIterationFunc is used to implement and extend rule group +// evaluation iteration logic. It is configured in Group.evalIterationFunc, +// and periodically invoked at each group evaluation interval to +// evaluate the rules in the group at that point in time. +// DefaultEvalIterationFunc is the default implementation. +type GroupEvalIterationFunc func(ctx context.Context, g *Group, evalTimestamp time.Time) + +type GroupOptions struct { + Name, File string + Interval time.Duration + Limit int + Rules []Rule + ShouldRestore bool + Opts *ManagerOptions + done chan struct{} + EvalIterationFunc GroupEvalIterationFunc +} + +// NewGroup makes a new Group with the given name, options, and rules. +func NewGroup(o GroupOptions) *Group { + metrics := o.Opts.Metrics + if metrics == nil { + metrics = NewGroupMetrics(o.Opts.Registerer) + } + + key := GroupKey(o.File, o.Name) + metrics.IterationsMissed.WithLabelValues(key) + metrics.IterationsScheduled.WithLabelValues(key) + metrics.EvalTotal.WithLabelValues(key) + metrics.EvalFailures.WithLabelValues(key) + metrics.GroupLastEvalTime.WithLabelValues(key) + metrics.GroupLastDuration.WithLabelValues(key) + metrics.GroupRules.WithLabelValues(key).Set(float64(len(o.Rules))) + metrics.GroupSamples.WithLabelValues(key) + metrics.GroupInterval.WithLabelValues(key).Set(o.Interval.Seconds()) + + evalIterationFunc := o.EvalIterationFunc + if evalIterationFunc == nil { + evalIterationFunc = DefaultEvalIterationFunc + } + + return &Group{ + name: o.Name, + file: o.File, + interval: o.Interval, + limit: o.Limit, + rules: o.Rules, + shouldRestore: o.ShouldRestore, + opts: o.Opts, + seriesInPreviousEval: make([]map[string]labels.Labels, len(o.Rules)), + done: make(chan struct{}), + managerDone: o.done, + terminated: make(chan struct{}), + logger: log.With(o.Opts.Logger, "file", o.File, "group", o.Name), + metrics: metrics, + evalIterationFunc: evalIterationFunc, + } +} + +// Name returns the group name. +func (g *Group) Name() string { return g.name } + +// File returns the group's file. +func (g *Group) File() string { return g.file } + +// Rules returns the group's rules. +func (g *Group) Rules() []Rule { return g.rules } + +// Queryable returns the group's querable. +func (g *Group) Queryable() storage.Queryable { return g.opts.Queryable } + +// Context returns the group's context. +func (g *Group) Context() context.Context { return g.opts.Context } + +// Interval returns the group's interval. +func (g *Group) Interval() time.Duration { return g.interval } + +// Limit returns the group's limit. +func (g *Group) Limit() int { return g.limit } + +func (g *Group) Logger() log.Logger { return g.logger } + +func (g *Group) run(ctx context.Context) { + defer close(g.terminated) + + // Wait an initial amount to have consistently slotted intervals. + evalTimestamp := g.EvalTimestamp(time.Now().UnixNano()).Add(g.interval) + select { + case <-time.After(time.Until(evalTimestamp)): + case <-g.done: + return + } + + ctx = promql.NewOriginContext(ctx, map[string]interface{}{ + "ruleGroup": map[string]string{ + "file": g.File(), + "name": g.Name(), + }, + }) + + // The assumption here is that since the ticker was started after having + // waited for `evalTimestamp` to pass, the ticks will trigger soon + // after each `evalTimestamp + N * g.interval` occurrence. + tick := time.NewTicker(g.interval) + defer tick.Stop() + + defer func() { + if !g.markStale { + return + } + go func(now time.Time) { + for _, rule := range g.seriesInPreviousEval { + for _, r := range rule { + g.staleSeries = append(g.staleSeries, r) + } + } + // That can be garbage collected at this point. + g.seriesInPreviousEval = nil + // Wait for 2 intervals to give the opportunity to renamed rules + // to insert new series in the tsdb. At this point if there is a + // renamed rule, it should already be started. + select { + case <-g.managerDone: + case <-time.After(2 * g.interval): + g.cleanupStaleSeries(ctx, now) + } + }(time.Now()) + }() + + g.evalIterationFunc(ctx, g, evalTimestamp) + if g.shouldRestore { + // If we have to restore, we wait for another Eval to finish. + // The reason behind this is, during first eval (or before it) + // we might not have enough data scraped, and recording rules would not + // have updated the latest values, on which some alerts might depend. + select { + case <-g.done: + return + case <-tick.C: + missed := (time.Since(evalTimestamp) / g.interval) - 1 + if missed > 0 { + g.metrics.IterationsMissed.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) + g.metrics.IterationsScheduled.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) + } + evalTimestamp = evalTimestamp.Add((missed + 1) * g.interval) + g.evalIterationFunc(ctx, g, evalTimestamp) + } + + g.RestoreForState(time.Now()) + g.shouldRestore = false + } + + for { + select { + case <-g.done: + return + default: + select { + case <-g.done: + return + case <-tick.C: + missed := (time.Since(evalTimestamp) / g.interval) - 1 + if missed > 0 { + g.metrics.IterationsMissed.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) + g.metrics.IterationsScheduled.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) + } + evalTimestamp = evalTimestamp.Add((missed + 1) * g.interval) + + g.evalIterationFunc(ctx, g, evalTimestamp) + } + } + } +} + +func (g *Group) stop() { + close(g.done) + <-g.terminated +} + +func (g *Group) hash() uint64 { + l := labels.New( + labels.Label{Name: "name", Value: g.name}, + labels.Label{Name: "file", Value: g.file}, + ) + return l.Hash() +} + +// AlertingRules returns the list of the group's alerting rules. +func (g *Group) AlertingRules() []*AlertingRule { + g.mtx.Lock() + defer g.mtx.Unlock() + + var alerts []*AlertingRule + for _, rule := range g.rules { + if alertingRule, ok := rule.(*AlertingRule); ok { + alerts = append(alerts, alertingRule) + } + } + slices.SortFunc(alerts, func(a, b *AlertingRule) int { + if a.State() == b.State() { + return strings.Compare(a.Name(), b.Name()) + } + return int(b.State() - a.State()) + }) + return alerts +} + +// HasAlertingRules returns true if the group contains at least one AlertingRule. +func (g *Group) HasAlertingRules() bool { + g.mtx.Lock() + defer g.mtx.Unlock() + + for _, rule := range g.rules { + if _, ok := rule.(*AlertingRule); ok { + return true + } + } + return false +} + +// GetEvaluationTime returns the time in seconds it took to evaluate the rule group. +func (g *Group) GetEvaluationTime() time.Duration { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.evaluationTime +} + +// setEvaluationTime sets the time in seconds the last evaluation took. +func (g *Group) setEvaluationTime(dur time.Duration) { + g.metrics.GroupLastDuration.WithLabelValues(GroupKey(g.file, g.name)).Set(dur.Seconds()) + + g.mtx.Lock() + defer g.mtx.Unlock() + g.evaluationTime = dur +} + +// GetLastEvaluation returns the time the last evaluation of the rule group took place. +func (g *Group) GetLastEvaluation() time.Time { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.lastEvaluation +} + +// setLastEvaluation updates evaluationTimestamp to the timestamp of when the rule group was last evaluated. +func (g *Group) setLastEvaluation(ts time.Time) { + g.metrics.GroupLastEvalTime.WithLabelValues(GroupKey(g.file, g.name)).Set(float64(ts.UnixNano()) / 1e9) + + g.mtx.Lock() + defer g.mtx.Unlock() + g.lastEvaluation = ts +} + +// GetLastEvalTimestamp returns the timestamp of the last evaluation. +func (g *Group) GetLastEvalTimestamp() time.Time { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.lastEvalTimestamp +} + +// setLastEvalTimestamp updates lastEvalTimestamp to the timestamp of the last evaluation. +func (g *Group) setLastEvalTimestamp(ts time.Time) { + g.mtx.Lock() + defer g.mtx.Unlock() + g.lastEvalTimestamp = ts +} + +// EvalTimestamp returns the immediately preceding consistently slotted evaluation time. +func (g *Group) EvalTimestamp(startTime int64) time.Time { + var ( + offset = int64(g.hash() % uint64(g.interval)) + + // This group's evaluation times differ from the perfect time intervals by `offset` nanoseconds. + // But we can only use `% interval` to align with the interval. And `% interval` will always + // align with the perfect time intervals, instead of this group's. Because of this we add + // `offset` _after_ aligning with the perfect time interval. + // + // There can be cases where adding `offset` to the perfect evaluation time can yield a + // timestamp in the future, which is not what EvalTimestamp should do. + // So we subtract one `offset` to make sure that `now - (now % interval) + offset` gives an + // evaluation time in the past. + adjNow = startTime - offset + + // Adjust to perfect evaluation intervals. + base = adjNow - (adjNow % int64(g.interval)) + + // Add one offset to randomize the evaluation times of this group. + next = base + offset + ) + + return time.Unix(0, next).UTC() +} + +func nameAndLabels(rule Rule) string { + return rule.Name() + rule.Labels().String() +} + +// CopyState copies the alerting rule and staleness related state from the given group. +// +// Rules are matched based on their name and labels. If there are duplicates, the +// first is matched with the first, second with the second etc. +func (g *Group) CopyState(from *Group) { + g.evaluationTime = from.evaluationTime + g.lastEvaluation = from.lastEvaluation + + ruleMap := make(map[string][]int, len(from.rules)) + + for fi, fromRule := range from.rules { + nameAndLabels := nameAndLabels(fromRule) + l := ruleMap[nameAndLabels] + ruleMap[nameAndLabels] = append(l, fi) + } + + for i, rule := range g.rules { + nameAndLabels := nameAndLabels(rule) + indexes := ruleMap[nameAndLabels] + if len(indexes) == 0 { + continue + } + fi := indexes[0] + g.seriesInPreviousEval[i] = from.seriesInPreviousEval[fi] + ruleMap[nameAndLabels] = indexes[1:] + + ar, ok := rule.(*AlertingRule) + if !ok { + continue + } + far, ok := from.rules[fi].(*AlertingRule) + if !ok { + continue + } + + for fp, a := range far.active { + ar.active[fp] = a + } + } + + // Handle deleted and unmatched duplicate rules. + g.staleSeries = from.staleSeries + for fi, fromRule := range from.rules { + nameAndLabels := nameAndLabels(fromRule) + l := ruleMap[nameAndLabels] + if len(l) != 0 { + for _, series := range from.seriesInPreviousEval[fi] { + g.staleSeries = append(g.staleSeries, series) + } + } + } +} + +// Eval runs a single evaluation cycle in which all rules are evaluated sequentially. +func (g *Group) Eval(ctx context.Context, ts time.Time) { + var samplesTotal float64 + for i, rule := range g.rules { + select { + case <-g.done: + return + default: + } + + func(i int, rule Rule) { + logger := log.WithPrefix(g.logger, "name", rule.Name(), "index", i) + ctx, sp := otel.Tracer("").Start(ctx, "rule") + sp.SetAttributes(attribute.String("name", rule.Name())) + defer func(t time.Time) { + sp.End() + + since := time.Since(t) + g.metrics.EvalDuration.Observe(since.Seconds()) + rule.SetEvaluationDuration(since) + rule.SetEvaluationTimestamp(t) + }(time.Now()) + + if sp.SpanContext().IsSampled() && sp.SpanContext().HasTraceID() { + logger = log.WithPrefix(logger, "traceID", sp.SpanContext().TraceID()) + } + + g.metrics.EvalTotal.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() + + vector, err := rule.Eval(ctx, ts, g.opts.QueryFunc, g.opts.ExternalURL, g.Limit()) + if err != nil { + rule.SetHealth(HealthBad) + rule.SetLastError(err) + sp.SetStatus(codes.Error, err.Error()) + g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() + + // Canceled queries are intentional termination of queries. This normally + // happens on shutdown and thus we skip logging of any errors here. + var eqc promql.ErrQueryCanceled + if !errors.As(err, &eqc) { + level.Warn(logger).Log("msg", "Evaluating rule failed", "rule", rule, "err", err) + } + return + } + rule.SetHealth(HealthGood) + rule.SetLastError(nil) + samplesTotal += float64(len(vector)) + + if ar, ok := rule.(*AlertingRule); ok { + ar.sendAlerts(ctx, ts, g.opts.ResendDelay, g.interval, g.opts.NotifyFunc) + } + var ( + numOutOfOrder = 0 + numTooOld = 0 + numDuplicates = 0 + ) + + app := g.opts.Appendable.Appender(ctx) + seriesReturned := make(map[string]labels.Labels, len(g.seriesInPreviousEval[i])) + defer func() { + if err := app.Commit(); err != nil { + rule.SetHealth(HealthBad) + rule.SetLastError(err) + sp.SetStatus(codes.Error, err.Error()) + g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() + + level.Warn(logger).Log("msg", "Rule sample appending failed", "err", err) + return + } + g.seriesInPreviousEval[i] = seriesReturned + }() + + for _, s := range vector { + if s.H != nil { + _, err = app.AppendHistogram(0, s.Metric, s.T, nil, s.H) + } else { + _, err = app.Append(0, s.Metric, s.T, s.F) + } + + if err != nil { + rule.SetHealth(HealthBad) + rule.SetLastError(err) + sp.SetStatus(codes.Error, err.Error()) + unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } + switch { + case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample): + numOutOfOrder++ + level.Debug(logger).Log("msg", "Rule evaluation result discarded", "err", err, "sample", s) + case errors.Is(unwrappedErr, storage.ErrTooOldSample): + numTooOld++ + level.Debug(logger).Log("msg", "Rule evaluation result discarded", "err", err, "sample", s) + case errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): + numDuplicates++ + level.Debug(logger).Log("msg", "Rule evaluation result discarded", "err", err, "sample", s) + default: + level.Warn(logger).Log("msg", "Rule evaluation result discarded", "err", err, "sample", s) + } + } else { + buf := [1024]byte{} + seriesReturned[string(s.Metric.Bytes(buf[:]))] = s.Metric + } + } + if numOutOfOrder > 0 { + level.Warn(logger).Log("msg", "Error on ingesting out-of-order result from rule evaluation", "numDropped", numOutOfOrder) + } + if numTooOld > 0 { + level.Warn(logger).Log("msg", "Error on ingesting too old result from rule evaluation", "numDropped", numTooOld) + } + if numDuplicates > 0 { + level.Warn(logger).Log("msg", "Error on ingesting results from rule evaluation with different value but same timestamp", "numDropped", numDuplicates) + } + + for metric, lset := range g.seriesInPreviousEval[i] { + if _, ok := seriesReturned[metric]; !ok { + // Series no longer exposed, mark it stale. + _, err = app.Append(0, lset, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN)) + unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } + switch { + case unwrappedErr == nil: + case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), + errors.Is(unwrappedErr, storage.ErrTooOldSample), + errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): + // Do not count these in logging, as this is expected if series + // is exposed from a different rule. + default: + level.Warn(logger).Log("msg", "Adding stale sample failed", "sample", lset.String(), "err", err) + } + } + } + }(i, rule) + } + if g.metrics != nil { + g.metrics.GroupSamples.WithLabelValues(GroupKey(g.File(), g.Name())).Set(samplesTotal) + } + g.cleanupStaleSeries(ctx, ts) +} + +func (g *Group) cleanupStaleSeries(ctx context.Context, ts time.Time) { + if len(g.staleSeries) == 0 { + return + } + app := g.opts.Appendable.Appender(ctx) + for _, s := range g.staleSeries { + // Rule that produced series no longer configured, mark it stale. + _, err := app.Append(0, s, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN)) + unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + unwrappedErr = err + } + switch { + case unwrappedErr == nil: + case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), + errors.Is(unwrappedErr, storage.ErrTooOldSample), + errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): + // Do not count these in logging, as this is expected if series + // is exposed from a different rule. + default: + level.Warn(g.logger).Log("msg", "Adding stale sample for previous configuration failed", "sample", s, "err", err) + } + } + if err := app.Commit(); err != nil { + level.Warn(g.logger).Log("msg", "Stale sample appending for previous configuration failed", "err", err) + } else { + g.staleSeries = nil + } +} + +// RestoreForState restores the 'for' state of the alerts +// by looking up last ActiveAt from storage. +func (g *Group) RestoreForState(ts time.Time) { + maxtMS := int64(model.TimeFromUnixNano(ts.UnixNano())) + // We allow restoration only if alerts were active before after certain time. + mint := ts.Add(-g.opts.OutageTolerance) + mintMS := int64(model.TimeFromUnixNano(mint.UnixNano())) + q, err := g.opts.Queryable.Querier(mintMS, maxtMS) + if err != nil { + level.Error(g.logger).Log("msg", "Failed to get Querier", "err", err) + return + } + defer func() { + if err := q.Close(); err != nil { + level.Error(g.logger).Log("msg", "Failed to close Querier", "err", err) + } + }() + + for _, rule := range g.Rules() { + alertRule, ok := rule.(*AlertingRule) + if !ok { + continue + } + + alertHoldDuration := alertRule.HoldDuration() + if alertHoldDuration < g.opts.ForGracePeriod { + // If alertHoldDuration is already less than grace period, we would not + // like to make it wait for `g.opts.ForGracePeriod` time before firing. + // Hence we skip restoration, which will make it wait for alertHoldDuration. + alertRule.SetRestored(true) + continue + } + + alertRule.ForEachActiveAlert(func(a *Alert) { + var s storage.Series + + s, err := alertRule.QueryforStateSeries(g.opts.Context, a, q) + if err != nil { + // Querier Warnings are ignored. We do not care unless we have an error. + level.Error(g.logger).Log( + "msg", "Failed to restore 'for' state", + labels.AlertName, alertRule.Name(), + "stage", "Select", + "err", err, + ) + return + } + + if s == nil { + return + } + + // Series found for the 'for' state. + var t int64 + var v float64 + it := s.Iterator(nil) + for it.Next() == chunkenc.ValFloat { + t, v = it.At() + } + if it.Err() != nil { + level.Error(g.logger).Log("msg", "Failed to restore 'for' state", + labels.AlertName, alertRule.Name(), "stage", "Iterator", "err", it.Err()) + return + } + if value.IsStaleNaN(v) { // Alert was not active. + return + } + + downAt := time.Unix(t/1000, 0).UTC() + restoredActiveAt := time.Unix(int64(v), 0).UTC() + timeSpentPending := downAt.Sub(restoredActiveAt) + timeRemainingPending := alertHoldDuration - timeSpentPending + + switch { + case timeRemainingPending <= 0: + // It means that alert was firing when prometheus went down. + // In the next Eval, the state of this alert will be set back to + // firing again if it's still firing in that Eval. + // Nothing to be done in this case. + case timeRemainingPending < g.opts.ForGracePeriod: + // (new) restoredActiveAt = (ts + m.opts.ForGracePeriod) - alertHoldDuration + // /* new firing time */ /* moving back by hold duration */ + // + // Proof of correctness: + // firingTime = restoredActiveAt.Add(alertHoldDuration) + // = ts + m.opts.ForGracePeriod - alertHoldDuration + alertHoldDuration + // = ts + m.opts.ForGracePeriod + // + // Time remaining to fire = firingTime.Sub(ts) + // = (ts + m.opts.ForGracePeriod) - ts + // = m.opts.ForGracePeriod + restoredActiveAt = ts.Add(g.opts.ForGracePeriod).Add(-alertHoldDuration) + default: + // By shifting ActiveAt to the future (ActiveAt + some_duration), + // the total pending time from the original ActiveAt + // would be `alertHoldDuration + some_duration`. + // Here, some_duration = downDuration. + downDuration := ts.Sub(downAt) + restoredActiveAt = restoredActiveAt.Add(downDuration) + } + + a.ActiveAt = restoredActiveAt + level.Debug(g.logger).Log("msg", "'for' state restored", + labels.AlertName, alertRule.Name(), "restored_time", a.ActiveAt.Format(time.RFC850), + "labels", a.Labels.String()) + }) + + alertRule.SetRestored(true) + } +} + +// Equals return if two groups are the same. +func (g *Group) Equals(ng *Group) bool { + if g.name != ng.name { + return false + } + + if g.file != ng.file { + return false + } + + if g.interval != ng.interval { + return false + } + + if g.limit != ng.limit { + return false + } + + if len(g.rules) != len(ng.rules) { + return false + } + + for i, gr := range g.rules { + if gr.String() != ng.rules[i].String() { + return false + } + } + + return true +} + +// GroupKey group names need not be unique across filenames. +func GroupKey(file, name string) string { + return file + ";" + name +} + +// Constants for instrumentation. +const namespace = "prometheus" + +// Metrics for rule evaluation. +type Metrics struct { + EvalDuration prometheus.Summary + IterationDuration prometheus.Summary + IterationsMissed *prometheus.CounterVec + IterationsScheduled *prometheus.CounterVec + EvalTotal *prometheus.CounterVec + EvalFailures *prometheus.CounterVec + GroupInterval *prometheus.GaugeVec + GroupLastEvalTime *prometheus.GaugeVec + GroupLastDuration *prometheus.GaugeVec + GroupRules *prometheus.GaugeVec + GroupSamples *prometheus.GaugeVec +} + +// NewGroupMetrics creates a new instance of Metrics and registers it with the provided registerer, +// if not nil. +func NewGroupMetrics(reg prometheus.Registerer) *Metrics { + m := &Metrics{ + EvalDuration: prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: "rule_evaluation_duration_seconds", + Help: "The duration for a rule to execute.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }), + IterationDuration: prometheus.NewSummary(prometheus.SummaryOpts{ + Namespace: namespace, + Name: "rule_group_duration_seconds", + Help: "The duration of rule group evaluations.", + Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, + }), + IterationsMissed: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "rule_group_iterations_missed_total", + Help: "The total number of rule group evaluations missed due to slow rule group evaluation.", + }, + []string{"rule_group"}, + ), + IterationsScheduled: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "rule_group_iterations_total", + Help: "The total number of scheduled rule group evaluations, whether executed or missed.", + }, + []string{"rule_group"}, + ), + EvalTotal: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "rule_evaluations_total", + Help: "The total number of rule evaluations.", + }, + []string{"rule_group"}, + ), + EvalFailures: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "rule_evaluation_failures_total", + Help: "The total number of rule evaluation failures.", + }, + []string{"rule_group"}, + ), + GroupInterval: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "rule_group_interval_seconds", + Help: "The interval of a rule group.", + }, + []string{"rule_group"}, + ), + GroupLastEvalTime: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "rule_group_last_evaluation_timestamp_seconds", + Help: "The timestamp of the last rule group evaluation in seconds.", + }, + []string{"rule_group"}, + ), + GroupLastDuration: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "rule_group_last_duration_seconds", + Help: "The duration of the last rule group evaluation.", + }, + []string{"rule_group"}, + ), + GroupRules: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "rule_group_rules", + Help: "The number of rules.", + }, + []string{"rule_group"}, + ), + GroupSamples: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "rule_group_last_evaluation_samples", + Help: "The number of samples returned during the last rule group evaluation.", + }, + []string{"rule_group"}, + ), + } + + if reg != nil { + reg.MustRegister( + m.EvalDuration, + m.IterationDuration, + m.IterationsMissed, + m.IterationsScheduled, + m.EvalTotal, + m.EvalFailures, + m.GroupInterval, + m.GroupLastEvalTime, + m.GroupLastDuration, + m.GroupRules, + m.GroupSamples, + ) + } + + return m +} diff --git a/vendor/github.com/prometheus/prometheus/rules/manager.go b/vendor/github.com/prometheus/prometheus/rules/manager.go index 4e10d39c7d..ed4d42ebad 100644 --- a/vendor/github.com/prometheus/prometheus/rules/manager.go +++ b/vendor/github.com/prometheus/prometheus/rules/manager.go @@ -17,7 +17,6 @@ import ( "context" "errors" "fmt" - "math" "net/url" "strings" "sync" @@ -26,162 +25,17 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" "golang.org/x/exp/slices" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" - "github.com/prometheus/prometheus/model/timestamp" - "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/util/strutil" ) -// RuleHealth describes the health state of a rule. -type RuleHealth string - -// The possible health states of a rule based on the last execution. -const ( - HealthUnknown RuleHealth = "unknown" - HealthGood RuleHealth = "ok" - HealthBad RuleHealth = "err" -) - -// Constants for instrumentation. -const namespace = "prometheus" - -// Metrics for rule evaluation. -type Metrics struct { - EvalDuration prometheus.Summary - IterationDuration prometheus.Summary - IterationsMissed *prometheus.CounterVec - IterationsScheduled *prometheus.CounterVec - EvalTotal *prometheus.CounterVec - EvalFailures *prometheus.CounterVec - GroupInterval *prometheus.GaugeVec - GroupLastEvalTime *prometheus.GaugeVec - GroupLastDuration *prometheus.GaugeVec - GroupRules *prometheus.GaugeVec - GroupSamples *prometheus.GaugeVec -} - -// NewGroupMetrics creates a new instance of Metrics and registers it with the provided registerer, -// if not nil. -func NewGroupMetrics(reg prometheus.Registerer) *Metrics { - m := &Metrics{ - EvalDuration: prometheus.NewSummary( - prometheus.SummaryOpts{ - Namespace: namespace, - Name: "rule_evaluation_duration_seconds", - Help: "The duration for a rule to execute.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }), - IterationDuration: prometheus.NewSummary(prometheus.SummaryOpts{ - Namespace: namespace, - Name: "rule_group_duration_seconds", - Help: "The duration of rule group evaluations.", - Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, - }), - IterationsMissed: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "rule_group_iterations_missed_total", - Help: "The total number of rule group evaluations missed due to slow rule group evaluation.", - }, - []string{"rule_group"}, - ), - IterationsScheduled: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "rule_group_iterations_total", - Help: "The total number of scheduled rule group evaluations, whether executed or missed.", - }, - []string{"rule_group"}, - ), - EvalTotal: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "rule_evaluations_total", - Help: "The total number of rule evaluations.", - }, - []string{"rule_group"}, - ), - EvalFailures: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: "rule_evaluation_failures_total", - Help: "The total number of rule evaluation failures.", - }, - []string{"rule_group"}, - ), - GroupInterval: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "rule_group_interval_seconds", - Help: "The interval of a rule group.", - }, - []string{"rule_group"}, - ), - GroupLastEvalTime: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "rule_group_last_evaluation_timestamp_seconds", - Help: "The timestamp of the last rule group evaluation in seconds.", - }, - []string{"rule_group"}, - ), - GroupLastDuration: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "rule_group_last_duration_seconds", - Help: "The duration of the last rule group evaluation.", - }, - []string{"rule_group"}, - ), - GroupRules: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "rule_group_rules", - Help: "The number of rules.", - }, - []string{"rule_group"}, - ), - GroupSamples: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: "rule_group_last_evaluation_samples", - Help: "The number of samples returned during the last rule group evaluation.", - }, - []string{"rule_group"}, - ), - } - - if reg != nil { - reg.MustRegister( - m.EvalDuration, - m.IterationDuration, - m.IterationsMissed, - m.IterationsScheduled, - m.EvalTotal, - m.EvalFailures, - m.GroupInterval, - m.GroupLastEvalTime, - m.GroupLastDuration, - m.GroupRules, - m.GroupSamples, - ) - } - - return m -} - // QueryFunc processes PromQL queries. type QueryFunc func(ctx context.Context, q string, t time.Time) (promql.Vector, error) @@ -213,241 +67,6 @@ func EngineQueryFunc(engine *promql.Engine, q storage.Queryable) QueryFunc { } } -// A Rule encapsulates a vector expression which is evaluated at a specified -// interval and acted upon (currently either recorded or used for alerting). -type Rule interface { - Name() string - // Labels of the rule. - Labels() labels.Labels - // eval evaluates the rule, including any associated recording or alerting actions. - Eval(context.Context, time.Time, QueryFunc, *url.URL, int) (promql.Vector, error) - // String returns a human-readable string representation of the rule. - String() string - // Query returns the rule query expression. - Query() parser.Expr - // SetLastErr sets the current error experienced by the rule. - SetLastError(error) - // LastErr returns the last error experienced by the rule. - LastError() error - // SetHealth sets the current health of the rule. - SetHealth(RuleHealth) - // Health returns the current health of the rule. - Health() RuleHealth - SetEvaluationDuration(time.Duration) - // GetEvaluationDuration returns last evaluation duration. - // NOTE: Used dynamically by rules.html template. - GetEvaluationDuration() time.Duration - SetEvaluationTimestamp(time.Time) - // GetEvaluationTimestamp returns last evaluation timestamp. - // NOTE: Used dynamically by rules.html template. - GetEvaluationTimestamp() time.Time -} - -// Group is a set of rules that have a logical relation. -type Group struct { - name string - file string - interval time.Duration - limit int - rules []Rule - seriesInPreviousEval []map[string]labels.Labels // One per Rule. - staleSeries []labels.Labels - opts *ManagerOptions - mtx sync.Mutex - evaluationTime time.Duration - lastEvaluation time.Time // Wall-clock time of most recent evaluation. - lastEvalTimestamp time.Time // Time slot used for most recent evaluation. - - shouldRestore bool - - markStale bool - done chan struct{} - terminated chan struct{} - managerDone chan struct{} - - logger log.Logger - - metrics *Metrics - - // Rule group evaluation iteration function, - // defaults to DefaultEvalIterationFunc. - evalIterationFunc GroupEvalIterationFunc -} - -// GroupEvalIterationFunc is used to implement and extend rule group -// evaluation iteration logic. It is configured in Group.evalIterationFunc, -// and periodically invoked at each group evaluation interval to -// evaluate the rules in the group at that point in time. -// DefaultEvalIterationFunc is the default implementation. -type GroupEvalIterationFunc func(ctx context.Context, g *Group, evalTimestamp time.Time) - -type GroupOptions struct { - Name, File string - Interval time.Duration - Limit int - Rules []Rule - ShouldRestore bool - Opts *ManagerOptions - done chan struct{} - EvalIterationFunc GroupEvalIterationFunc -} - -// NewGroup makes a new Group with the given name, options, and rules. -func NewGroup(o GroupOptions) *Group { - metrics := o.Opts.Metrics - if metrics == nil { - metrics = NewGroupMetrics(o.Opts.Registerer) - } - - key := GroupKey(o.File, o.Name) - metrics.IterationsMissed.WithLabelValues(key) - metrics.IterationsScheduled.WithLabelValues(key) - metrics.EvalTotal.WithLabelValues(key) - metrics.EvalFailures.WithLabelValues(key) - metrics.GroupLastEvalTime.WithLabelValues(key) - metrics.GroupLastDuration.WithLabelValues(key) - metrics.GroupRules.WithLabelValues(key).Set(float64(len(o.Rules))) - metrics.GroupSamples.WithLabelValues(key) - metrics.GroupInterval.WithLabelValues(key).Set(o.Interval.Seconds()) - - evalIterationFunc := o.EvalIterationFunc - if evalIterationFunc == nil { - evalIterationFunc = DefaultEvalIterationFunc - } - - return &Group{ - name: o.Name, - file: o.File, - interval: o.Interval, - limit: o.Limit, - rules: o.Rules, - shouldRestore: o.ShouldRestore, - opts: o.Opts, - seriesInPreviousEval: make([]map[string]labels.Labels, len(o.Rules)), - done: make(chan struct{}), - managerDone: o.done, - terminated: make(chan struct{}), - logger: log.With(o.Opts.Logger, "file", o.File, "group", o.Name), - metrics: metrics, - evalIterationFunc: evalIterationFunc, - } -} - -// Name returns the group name. -func (g *Group) Name() string { return g.name } - -// File returns the group's file. -func (g *Group) File() string { return g.file } - -// Rules returns the group's rules. -func (g *Group) Rules() []Rule { return g.rules } - -// Queryable returns the group's querable. -func (g *Group) Queryable() storage.Queryable { return g.opts.Queryable } - -// Context returns the group's context. -func (g *Group) Context() context.Context { return g.opts.Context } - -// Interval returns the group's interval. -func (g *Group) Interval() time.Duration { return g.interval } - -// Limit returns the group's limit. -func (g *Group) Limit() int { return g.limit } - -func (g *Group) Logger() log.Logger { return g.logger } - -func (g *Group) run(ctx context.Context) { - defer close(g.terminated) - - // Wait an initial amount to have consistently slotted intervals. - evalTimestamp := g.EvalTimestamp(time.Now().UnixNano()).Add(g.interval) - select { - case <-time.After(time.Until(evalTimestamp)): - case <-g.done: - return - } - - ctx = promql.NewOriginContext(ctx, map[string]interface{}{ - "ruleGroup": map[string]string{ - "file": g.File(), - "name": g.Name(), - }, - }) - - // The assumption here is that since the ticker was started after having - // waited for `evalTimestamp` to pass, the ticks will trigger soon - // after each `evalTimestamp + N * g.interval` occurrence. - tick := time.NewTicker(g.interval) - defer tick.Stop() - - defer func() { - if !g.markStale { - return - } - go func(now time.Time) { - for _, rule := range g.seriesInPreviousEval { - for _, r := range rule { - g.staleSeries = append(g.staleSeries, r) - } - } - // That can be garbage collected at this point. - g.seriesInPreviousEval = nil - // Wait for 2 intervals to give the opportunity to renamed rules - // to insert new series in the tsdb. At this point if there is a - // renamed rule, it should already be started. - select { - case <-g.managerDone: - case <-time.After(2 * g.interval): - g.cleanupStaleSeries(ctx, now) - } - }(time.Now()) - }() - - g.evalIterationFunc(ctx, g, evalTimestamp) - if g.shouldRestore { - // If we have to restore, we wait for another Eval to finish. - // The reason behind this is, during first eval (or before it) - // we might not have enough data scraped, and recording rules would not - // have updated the latest values, on which some alerts might depend. - select { - case <-g.done: - return - case <-tick.C: - missed := (time.Since(evalTimestamp) / g.interval) - 1 - if missed > 0 { - g.metrics.IterationsMissed.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) - g.metrics.IterationsScheduled.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) - } - evalTimestamp = evalTimestamp.Add((missed + 1) * g.interval) - g.evalIterationFunc(ctx, g, evalTimestamp) - } - - g.RestoreForState(time.Now()) - g.shouldRestore = false - } - - for { - select { - case <-g.done: - return - default: - select { - case <-g.done: - return - case <-tick.C: - missed := (time.Since(evalTimestamp) / g.interval) - 1 - if missed > 0 { - g.metrics.IterationsMissed.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) - g.metrics.IterationsScheduled.WithLabelValues(GroupKey(g.file, g.name)).Add(float64(missed)) - } - evalTimestamp = evalTimestamp.Add((missed + 1) * g.interval) - - g.evalIterationFunc(ctx, g, evalTimestamp) - } - } - } -} - // DefaultEvalIterationFunc is the default implementation of // GroupEvalIterationFunc that is periodically invoked to evaluate the rules // in a group at a given point in time and updates Group state and metrics @@ -467,491 +86,6 @@ func DefaultEvalIterationFunc(ctx context.Context, g *Group, evalTimestamp time. g.setLastEvalTimestamp(evalTimestamp) } -func (g *Group) stop() { - close(g.done) - <-g.terminated -} - -func (g *Group) hash() uint64 { - l := labels.New( - labels.Label{Name: "name", Value: g.name}, - labels.Label{Name: "file", Value: g.file}, - ) - return l.Hash() -} - -// AlertingRules returns the list of the group's alerting rules. -func (g *Group) AlertingRules() []*AlertingRule { - g.mtx.Lock() - defer g.mtx.Unlock() - - var alerts []*AlertingRule - for _, rule := range g.rules { - if alertingRule, ok := rule.(*AlertingRule); ok { - alerts = append(alerts, alertingRule) - } - } - slices.SortFunc(alerts, func(a, b *AlertingRule) int { - if a.State() == b.State() { - return strings.Compare(a.Name(), b.Name()) - } - return int(b.State() - a.State()) - }) - return alerts -} - -// HasAlertingRules returns true if the group contains at least one AlertingRule. -func (g *Group) HasAlertingRules() bool { - g.mtx.Lock() - defer g.mtx.Unlock() - - for _, rule := range g.rules { - if _, ok := rule.(*AlertingRule); ok { - return true - } - } - return false -} - -// GetEvaluationTime returns the time in seconds it took to evaluate the rule group. -func (g *Group) GetEvaluationTime() time.Duration { - g.mtx.Lock() - defer g.mtx.Unlock() - return g.evaluationTime -} - -// setEvaluationTime sets the time in seconds the last evaluation took. -func (g *Group) setEvaluationTime(dur time.Duration) { - g.metrics.GroupLastDuration.WithLabelValues(GroupKey(g.file, g.name)).Set(dur.Seconds()) - - g.mtx.Lock() - defer g.mtx.Unlock() - g.evaluationTime = dur -} - -// GetLastEvaluation returns the time the last evaluation of the rule group took place. -func (g *Group) GetLastEvaluation() time.Time { - g.mtx.Lock() - defer g.mtx.Unlock() - return g.lastEvaluation -} - -// setLastEvaluation updates evaluationTimestamp to the timestamp of when the rule group was last evaluated. -func (g *Group) setLastEvaluation(ts time.Time) { - g.metrics.GroupLastEvalTime.WithLabelValues(GroupKey(g.file, g.name)).Set(float64(ts.UnixNano()) / 1e9) - - g.mtx.Lock() - defer g.mtx.Unlock() - g.lastEvaluation = ts -} - -// GetLastEvalTimestamp returns the timestamp of the last evaluation. -func (g *Group) GetLastEvalTimestamp() time.Time { - g.mtx.Lock() - defer g.mtx.Unlock() - return g.lastEvalTimestamp -} - -// setLastEvalTimestamp updates lastEvalTimestamp to the timestamp of the last evaluation. -func (g *Group) setLastEvalTimestamp(ts time.Time) { - g.mtx.Lock() - defer g.mtx.Unlock() - g.lastEvalTimestamp = ts -} - -// EvalTimestamp returns the immediately preceding consistently slotted evaluation time. -func (g *Group) EvalTimestamp(startTime int64) time.Time { - var ( - offset = int64(g.hash() % uint64(g.interval)) - - // This group's evaluation times differ from the perfect time intervals by `offset` nanoseconds. - // But we can only use `% interval` to align with the interval. And `% interval` will always - // align with the perfect time intervals, instead of this group's. Because of this we add - // `offset` _after_ aligning with the perfect time interval. - // - // There can be cases where adding `offset` to the perfect evaluation time can yield a - // timestamp in the future, which is not what EvalTimestamp should do. - // So we subtract one `offset` to make sure that `now - (now % interval) + offset` gives an - // evaluation time in the past. - adjNow = startTime - offset - - // Adjust to perfect evaluation intervals. - base = adjNow - (adjNow % int64(g.interval)) - - // Add one offset to randomize the evaluation times of this group. - next = base + offset - ) - - return time.Unix(0, next).UTC() -} - -func nameAndLabels(rule Rule) string { - return rule.Name() + rule.Labels().String() -} - -// CopyState copies the alerting rule and staleness related state from the given group. -// -// Rules are matched based on their name and labels. If there are duplicates, the -// first is matched with the first, second with the second etc. -func (g *Group) CopyState(from *Group) { - g.evaluationTime = from.evaluationTime - g.lastEvaluation = from.lastEvaluation - - ruleMap := make(map[string][]int, len(from.rules)) - - for fi, fromRule := range from.rules { - nameAndLabels := nameAndLabels(fromRule) - l := ruleMap[nameAndLabels] - ruleMap[nameAndLabels] = append(l, fi) - } - - for i, rule := range g.rules { - nameAndLabels := nameAndLabels(rule) - indexes := ruleMap[nameAndLabels] - if len(indexes) == 0 { - continue - } - fi := indexes[0] - g.seriesInPreviousEval[i] = from.seriesInPreviousEval[fi] - ruleMap[nameAndLabels] = indexes[1:] - - ar, ok := rule.(*AlertingRule) - if !ok { - continue - } - far, ok := from.rules[fi].(*AlertingRule) - if !ok { - continue - } - - for fp, a := range far.active { - ar.active[fp] = a - } - } - - // Handle deleted and unmatched duplicate rules. - g.staleSeries = from.staleSeries - for fi, fromRule := range from.rules { - nameAndLabels := nameAndLabels(fromRule) - l := ruleMap[nameAndLabels] - if len(l) != 0 { - for _, series := range from.seriesInPreviousEval[fi] { - g.staleSeries = append(g.staleSeries, series) - } - } - } -} - -// Eval runs a single evaluation cycle in which all rules are evaluated sequentially. -func (g *Group) Eval(ctx context.Context, ts time.Time) { - var samplesTotal float64 - for i, rule := range g.rules { - select { - case <-g.done: - return - default: - } - - func(i int, rule Rule) { - ctx, sp := otel.Tracer("").Start(ctx, "rule") - sp.SetAttributes(attribute.String("name", rule.Name())) - defer func(t time.Time) { - sp.End() - - since := time.Since(t) - g.metrics.EvalDuration.Observe(since.Seconds()) - rule.SetEvaluationDuration(since) - rule.SetEvaluationTimestamp(t) - }(time.Now()) - - g.metrics.EvalTotal.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() - - vector, err := rule.Eval(ctx, ts, g.opts.QueryFunc, g.opts.ExternalURL, g.Limit()) - if err != nil { - rule.SetHealth(HealthBad) - rule.SetLastError(err) - sp.SetStatus(codes.Error, err.Error()) - g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() - - // Canceled queries are intentional termination of queries. This normally - // happens on shutdown and thus we skip logging of any errors here. - var eqc promql.ErrQueryCanceled - if !errors.As(err, &eqc) { - level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Evaluating rule failed", "rule", rule, "err", err) - } - return - } - rule.SetHealth(HealthGood) - rule.SetLastError(nil) - samplesTotal += float64(len(vector)) - - if ar, ok := rule.(*AlertingRule); ok { - ar.sendAlerts(ctx, ts, g.opts.ResendDelay, g.interval, g.opts.NotifyFunc) - } - var ( - numOutOfOrder = 0 - numTooOld = 0 - numDuplicates = 0 - ) - - app := g.opts.Appendable.Appender(ctx) - seriesReturned := make(map[string]labels.Labels, len(g.seriesInPreviousEval[i])) - defer func() { - if err := app.Commit(); err != nil { - rule.SetHealth(HealthBad) - rule.SetLastError(err) - sp.SetStatus(codes.Error, err.Error()) - g.metrics.EvalFailures.WithLabelValues(GroupKey(g.File(), g.Name())).Inc() - - level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule sample appending failed", "err", err) - return - } - g.seriesInPreviousEval[i] = seriesReturned - }() - - for _, s := range vector { - if s.H != nil { - _, err = app.AppendHistogram(0, s.Metric, s.T, nil, s.H) - } else { - _, err = app.Append(0, s.Metric, s.T, s.F) - } - - if err != nil { - rule.SetHealth(HealthBad) - rule.SetLastError(err) - sp.SetStatus(codes.Error, err.Error()) - unwrappedErr := errors.Unwrap(err) - if unwrappedErr == nil { - unwrappedErr = err - } - switch { - case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample): - numOutOfOrder++ - level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s) - case errors.Is(unwrappedErr, storage.ErrTooOldSample): - numTooOld++ - level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s) - case errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): - numDuplicates++ - level.Debug(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s) - default: - level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Rule evaluation result discarded", "err", err, "sample", s) - } - } else { - buf := [1024]byte{} - seriesReturned[string(s.Metric.Bytes(buf[:]))] = s.Metric - } - } - if numOutOfOrder > 0 { - level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting out-of-order result from rule evaluation", "numDropped", numOutOfOrder) - } - if numTooOld > 0 { - level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting too old result from rule evaluation", "numDropped", numTooOld) - } - if numDuplicates > 0 { - level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Error on ingesting results from rule evaluation with different value but same timestamp", "numDropped", numDuplicates) - } - - for metric, lset := range g.seriesInPreviousEval[i] { - if _, ok := seriesReturned[metric]; !ok { - // Series no longer exposed, mark it stale. - _, err = app.Append(0, lset, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN)) - unwrappedErr := errors.Unwrap(err) - if unwrappedErr == nil { - unwrappedErr = err - } - switch { - case unwrappedErr == nil: - case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), - errors.Is(unwrappedErr, storage.ErrTooOldSample), - errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): - // Do not count these in logging, as this is expected if series - // is exposed from a different rule. - default: - level.Warn(g.logger).Log("name", rule.Name(), "index", i, "msg", "Adding stale sample failed", "sample", lset.String(), "err", err) - } - } - } - }(i, rule) - } - if g.metrics != nil { - g.metrics.GroupSamples.WithLabelValues(GroupKey(g.File(), g.Name())).Set(samplesTotal) - } - g.cleanupStaleSeries(ctx, ts) -} - -func (g *Group) cleanupStaleSeries(ctx context.Context, ts time.Time) { - if len(g.staleSeries) == 0 { - return - } - app := g.opts.Appendable.Appender(ctx) - for _, s := range g.staleSeries { - // Rule that produced series no longer configured, mark it stale. - _, err := app.Append(0, s, timestamp.FromTime(ts), math.Float64frombits(value.StaleNaN)) - unwrappedErr := errors.Unwrap(err) - if unwrappedErr == nil { - unwrappedErr = err - } - switch { - case unwrappedErr == nil: - case errors.Is(unwrappedErr, storage.ErrOutOfOrderSample), - errors.Is(unwrappedErr, storage.ErrTooOldSample), - errors.Is(unwrappedErr, storage.ErrDuplicateSampleForTimestamp): - // Do not count these in logging, as this is expected if series - // is exposed from a different rule. - default: - level.Warn(g.logger).Log("msg", "Adding stale sample for previous configuration failed", "sample", s, "err", err) - } - } - if err := app.Commit(); err != nil { - level.Warn(g.logger).Log("msg", "Stale sample appending for previous configuration failed", "err", err) - } else { - g.staleSeries = nil - } -} - -// RestoreForState restores the 'for' state of the alerts -// by looking up last ActiveAt from storage. -func (g *Group) RestoreForState(ts time.Time) { - maxtMS := int64(model.TimeFromUnixNano(ts.UnixNano())) - // We allow restoration only if alerts were active before after certain time. - mint := ts.Add(-g.opts.OutageTolerance) - mintMS := int64(model.TimeFromUnixNano(mint.UnixNano())) - q, err := g.opts.Queryable.Querier(mintMS, maxtMS) - if err != nil { - level.Error(g.logger).Log("msg", "Failed to get Querier", "err", err) - return - } - defer func() { - if err := q.Close(); err != nil { - level.Error(g.logger).Log("msg", "Failed to close Querier", "err", err) - } - }() - - for _, rule := range g.Rules() { - alertRule, ok := rule.(*AlertingRule) - if !ok { - continue - } - - alertHoldDuration := alertRule.HoldDuration() - if alertHoldDuration < g.opts.ForGracePeriod { - // If alertHoldDuration is already less than grace period, we would not - // like to make it wait for `g.opts.ForGracePeriod` time before firing. - // Hence we skip restoration, which will make it wait for alertHoldDuration. - alertRule.SetRestored(true) - continue - } - - alertRule.ForEachActiveAlert(func(a *Alert) { - var s storage.Series - - s, err := alertRule.QueryforStateSeries(g.opts.Context, a, q) - if err != nil { - // Querier Warnings are ignored. We do not care unless we have an error. - level.Error(g.logger).Log( - "msg", "Failed to restore 'for' state", - labels.AlertName, alertRule.Name(), - "stage", "Select", - "err", err, - ) - return - } - - if s == nil { - return - } - - // Series found for the 'for' state. - var t int64 - var v float64 - it := s.Iterator(nil) - for it.Next() == chunkenc.ValFloat { - t, v = it.At() - } - if it.Err() != nil { - level.Error(g.logger).Log("msg", "Failed to restore 'for' state", - labels.AlertName, alertRule.Name(), "stage", "Iterator", "err", it.Err()) - return - } - if value.IsStaleNaN(v) { // Alert was not active. - return - } - - downAt := time.Unix(t/1000, 0).UTC() - restoredActiveAt := time.Unix(int64(v), 0).UTC() - timeSpentPending := downAt.Sub(restoredActiveAt) - timeRemainingPending := alertHoldDuration - timeSpentPending - - switch { - case timeRemainingPending <= 0: - // It means that alert was firing when prometheus went down. - // In the next Eval, the state of this alert will be set back to - // firing again if it's still firing in that Eval. - // Nothing to be done in this case. - case timeRemainingPending < g.opts.ForGracePeriod: - // (new) restoredActiveAt = (ts + m.opts.ForGracePeriod) - alertHoldDuration - // /* new firing time */ /* moving back by hold duration */ - // - // Proof of correctness: - // firingTime = restoredActiveAt.Add(alertHoldDuration) - // = ts + m.opts.ForGracePeriod - alertHoldDuration + alertHoldDuration - // = ts + m.opts.ForGracePeriod - // - // Time remaining to fire = firingTime.Sub(ts) - // = (ts + m.opts.ForGracePeriod) - ts - // = m.opts.ForGracePeriod - restoredActiveAt = ts.Add(g.opts.ForGracePeriod).Add(-alertHoldDuration) - default: - // By shifting ActiveAt to the future (ActiveAt + some_duration), - // the total pending time from the original ActiveAt - // would be `alertHoldDuration + some_duration`. - // Here, some_duration = downDuration. - downDuration := ts.Sub(downAt) - restoredActiveAt = restoredActiveAt.Add(downDuration) - } - - a.ActiveAt = restoredActiveAt - level.Debug(g.logger).Log("msg", "'for' state restored", - labels.AlertName, alertRule.Name(), "restored_time", a.ActiveAt.Format(time.RFC850), - "labels", a.Labels.String()) - }) - - alertRule.SetRestored(true) - } -} - -// Equals return if two groups are the same. -func (g *Group) Equals(ng *Group) bool { - if g.name != ng.name { - return false - } - - if g.file != ng.file { - return false - } - - if g.interval != ng.interval { - return false - } - - if g.limit != ng.limit { - return false - } - - if len(g.rules) != len(ng.rules) { - return false - } - - for i, gr := range g.rules { - if gr.String() != ng.rules[i].String() { - return false - } - } - - return true -} - // The Manager manages recording and alerting rules. type Manager struct { opts *ManagerOptions @@ -1116,7 +250,7 @@ type GroupLoader interface { } // FileLoader is the default GroupLoader implementation. It defers to rulefmt.ParseFile -// and parser.ParseExpr +// and parser.ParseExpr. type FileLoader struct{} func (FileLoader) Load(identifier string) (*rulefmt.RuleGroups, []error) { @@ -1191,11 +325,6 @@ func (m *Manager) LoadGroups( return groups, nil } -// GroupKey group names need not be unique across filenames. -func GroupKey(file, name string) string { - return file + ";" + name -} - // RuleGroups returns the list of manager's rule groups. func (m *Manager) RuleGroups() []*Group { m.mtx.RLock() diff --git a/vendor/github.com/prometheus/prometheus/rules/rule.go b/vendor/github.com/prometheus/prometheus/rules/rule.go new file mode 100644 index 0000000000..a4a8c04459 --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/rules/rule.go @@ -0,0 +1,64 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rules + +import ( + "context" + "net/url" + "time" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/promql" + "github.com/prometheus/prometheus/promql/parser" +) + +// RuleHealth describes the health state of a rule. +type RuleHealth string + +// The possible health states of a rule based on the last execution. +const ( + HealthUnknown RuleHealth = "unknown" + HealthGood RuleHealth = "ok" + HealthBad RuleHealth = "err" +) + +// A Rule encapsulates a vector expression which is evaluated at a specified +// interval and acted upon (currently either recorded or used for alerting). +type Rule interface { + Name() string + // Labels of the rule. + Labels() labels.Labels + // Eval evaluates the rule, including any associated recording or alerting actions. + Eval(context.Context, time.Time, QueryFunc, *url.URL, int) (promql.Vector, error) + // String returns a human-readable string representation of the rule. + String() string + // Query returns the rule query expression. + Query() parser.Expr + // SetLastError sets the current error experienced by the rule. + SetLastError(error) + // LastError returns the last error experienced by the rule. + LastError() error + // SetHealth sets the current health of the rule. + SetHealth(RuleHealth) + // Health returns the current health of the rule. + Health() RuleHealth + SetEvaluationDuration(time.Duration) + // GetEvaluationDuration returns last evaluation duration. + // NOTE: Used dynamically by rules.html template. + GetEvaluationDuration() time.Duration + SetEvaluationTimestamp(time.Time) + // GetEvaluationTimestamp returns last evaluation timestamp. + // NOTE: Used dynamically by rules.html template. + GetEvaluationTimestamp() time.Time +} diff --git a/vendor/github.com/prometheus/prometheus/scrape/manager.go b/vendor/github.com/prometheus/prometheus/scrape/manager.go index 427b9f2be1..3b70e48a13 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/manager.go +++ b/vendor/github.com/prometheus/prometheus/scrape/manager.go @@ -14,6 +14,7 @@ package scrape import ( + "errors" "fmt" "hash/fnv" "reflect" @@ -22,7 +23,6 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" @@ -32,82 +32,23 @@ import ( "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/osutil" + "github.com/prometheus/prometheus/util/pool" ) -var targetMetadataCache = newMetadataMetricsCollector() - -// MetadataMetricsCollector is a Custom Collector for the metadata cache metrics. -type MetadataMetricsCollector struct { - CacheEntries *prometheus.Desc - CacheBytes *prometheus.Desc - - scrapeManager *Manager -} - -func newMetadataMetricsCollector() *MetadataMetricsCollector { - return &MetadataMetricsCollector{ - CacheEntries: prometheus.NewDesc( - "prometheus_target_metadata_cache_entries", - "Total number of metric metadata entries in the cache", - []string{"scrape_job"}, - nil, - ), - CacheBytes: prometheus.NewDesc( - "prometheus_target_metadata_cache_bytes", - "The number of bytes that are currently used for storing metric metadata in the cache", - []string{"scrape_job"}, - nil, - ), - } -} - -func (mc *MetadataMetricsCollector) registerManager(m *Manager) { - mc.scrapeManager = m -} - -// Describe sends the metrics descriptions to the channel. -func (mc *MetadataMetricsCollector) Describe(ch chan<- *prometheus.Desc) { - ch <- mc.CacheEntries - ch <- mc.CacheBytes -} - -// Collect creates and sends the metrics for the metadata cache. -func (mc *MetadataMetricsCollector) Collect(ch chan<- prometheus.Metric) { - if mc.scrapeManager == nil { - return - } - - for tset, targets := range mc.scrapeManager.TargetsActive() { - var size, length int - for _, t := range targets { - size += t.MetadataSize() - length += t.MetadataLength() - } - - ch <- prometheus.MustNewConstMetric( - mc.CacheEntries, - prometheus.GaugeValue, - float64(length), - tset, - ) - - ch <- prometheus.MustNewConstMetric( - mc.CacheBytes, - prometheus.GaugeValue, - float64(size), - tset, - ) - } -} - -// NewManager is the Manager constructor -func NewManager(o *Options, logger log.Logger, app storage.Appendable) *Manager { +// NewManager is the Manager constructor. +func NewManager(o *Options, logger log.Logger, app storage.Appendable, registerer prometheus.Registerer) (*Manager, error) { if o == nil { o = &Options{} } if logger == nil { logger = log.NewNopLogger() } + + sm, err := newScrapeMetrics(registerer) + if err != nil { + return nil, fmt.Errorf("failed to create scrape manager due to error: %w", err) + } + m := &Manager{ append: app, opts: o, @@ -116,10 +57,13 @@ func NewManager(o *Options, logger log.Logger, app storage.Appendable) *Manager scrapePools: make(map[string]*scrapePool), graceShut: make(chan struct{}), triggerReload: make(chan struct{}, 1), + metrics: sm, + buffers: pool.New(1e3, 100e6, 3, func(sz int) interface{} { return make([]byte, 0, sz) }), } - targetMetadataCache.registerManager(m) - return m + m.metrics.setTargetMetadataCacheGatherer(m) + + return m, nil } // Options are the configuration parameters to the scrape manager. @@ -132,9 +76,6 @@ type Options struct { // Option to enable the experimental in-memory metadata storage and append // metadata to the WAL. EnableMetadataStorage bool - // Option to enable protobuf negotiation with the client. Note that the client can already - // send protobuf without needing to enable this. - EnableProtobufNegotiation bool // Option to increase the interval used by scrape manager to throttle target groups updates. DiscoveryReloadInterval model.Duration @@ -155,8 +96,11 @@ type Manager struct { scrapeConfigs map[string]*config.ScrapeConfig scrapePools map[string]*scrapePool targetSets map[string][]*targetgroup.Group + buffers *pool.Pool triggerReload chan struct{} + + metrics *scrapeMetrics } // Run receives and saves target set updates and triggers the scraping loops reloading. @@ -214,8 +158,10 @@ func (m *Manager) reload() { level.Error(m.logger).Log("msg", "error reloading target set", "err", "invalid config id:"+setName) continue } - sp, err := newScrapePool(scrapeConfig, m.append, m.offsetSeed, log.With(m.logger, "scrape_pool", setName), m.opts) + m.metrics.targetScrapePools.Inc() + sp, err := newScrapePool(scrapeConfig, m.append, m.offsetSeed, log.With(m.logger, "scrape_pool", setName), m.buffers, m.opts, m.metrics) if err != nil { + m.metrics.targetScrapePoolsFailed.Inc() level.Error(m.logger).Log("msg", "error creating new scrape pool", "err", err, "scrape_pool", setName) continue } diff --git a/vendor/github.com/prometheus/prometheus/scrape/metrics.go b/vendor/github.com/prometheus/prometheus/scrape/metrics.go new file mode 100644 index 0000000000..d74143185b --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/scrape/metrics.go @@ -0,0 +1,307 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scrape + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" +) + +type scrapeMetrics struct { + // Used by Manager. + targetMetadataCache *MetadataMetricsCollector + targetScrapePools prometheus.Counter + targetScrapePoolsFailed prometheus.Counter + + // Used by scrapePool. + targetReloadIntervalLength *prometheus.SummaryVec + targetScrapePoolReloads prometheus.Counter + targetScrapePoolReloadsFailed prometheus.Counter + targetScrapePoolSyncsCounter *prometheus.CounterVec + targetScrapePoolExceededTargetLimit prometheus.Counter + targetScrapePoolTargetLimit *prometheus.GaugeVec + targetScrapePoolTargetsAdded *prometheus.GaugeVec + targetSyncIntervalLength *prometheus.SummaryVec + targetSyncFailed *prometheus.CounterVec + + // Used by targetScraper. + targetScrapeExceededBodySizeLimit prometheus.Counter + + // Used by scrapeCache. + targetScrapeCacheFlushForced prometheus.Counter + + // Used by scrapeLoop. + targetIntervalLength *prometheus.SummaryVec + targetScrapeSampleLimit prometheus.Counter + targetScrapeSampleDuplicate prometheus.Counter + targetScrapeSampleOutOfOrder prometheus.Counter + targetScrapeSampleOutOfBounds prometheus.Counter + targetScrapeExemplarOutOfOrder prometheus.Counter + targetScrapePoolExceededLabelLimits prometheus.Counter + targetScrapeNativeHistogramBucketLimit prometheus.Counter +} + +func newScrapeMetrics(reg prometheus.Registerer) (*scrapeMetrics, error) { + sm := &scrapeMetrics{} + + // Manager metrics. + sm.targetMetadataCache = &MetadataMetricsCollector{ + CacheEntries: prometheus.NewDesc( + "prometheus_target_metadata_cache_entries", + "Total number of metric metadata entries in the cache", + []string{"scrape_job"}, + nil, + ), + CacheBytes: prometheus.NewDesc( + "prometheus_target_metadata_cache_bytes", + "The number of bytes that are currently used for storing metric metadata in the cache", + []string{"scrape_job"}, + nil, + ), + // TargetsGatherer should be set later, because it's a circular dependency. + // newScrapeMetrics() is called by NewManager(), while also TargetsGatherer is the new Manager. + } + + sm.targetScrapePools = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrape_pools_total", + Help: "Total number of scrape pool creation attempts.", + }, + ) + sm.targetScrapePoolsFailed = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrape_pools_failed_total", + Help: "Total number of scrape pool creations that failed.", + }, + ) + + // Used by scrapePool. + sm.targetReloadIntervalLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "prometheus_target_reload_length_seconds", + Help: "Actual interval to reload the scrape pool with a given configuration.", + Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, + }, + []string{"interval"}, + ) + sm.targetScrapePoolReloads = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrape_pool_reloads_total", + Help: "Total number of scrape pool reloads.", + }, + ) + sm.targetScrapePoolReloadsFailed = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrape_pool_reloads_failed_total", + Help: "Total number of failed scrape pool reloads.", + }, + ) + sm.targetScrapePoolExceededTargetLimit = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrape_pool_exceeded_target_limit_total", + Help: "Total number of times scrape pools hit the target limit, during sync or config reload.", + }, + ) + sm.targetScrapePoolTargetLimit = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prometheus_target_scrape_pool_target_limit", + Help: "Maximum number of targets allowed in this scrape pool.", + }, + []string{"scrape_job"}, + ) + sm.targetScrapePoolTargetsAdded = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prometheus_target_scrape_pool_targets", + Help: "Current number of targets in this scrape pool.", + }, + []string{"scrape_job"}, + ) + sm.targetScrapePoolSyncsCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "prometheus_target_scrape_pool_sync_total", + Help: "Total number of syncs that were executed on a scrape pool.", + }, + []string{"scrape_job"}, + ) + sm.targetSyncIntervalLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "prometheus_target_sync_length_seconds", + Help: "Actual interval to sync the scrape pool.", + Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, + }, + []string{"scrape_job"}, + ) + sm.targetSyncFailed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "prometheus_target_sync_failed_total", + Help: "Total number of target sync failures.", + }, + []string{"scrape_job"}, + ) + + // Used by targetScraper. + sm.targetScrapeExceededBodySizeLimit = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_exceeded_body_size_limit_total", + Help: "Total number of scrapes that hit the body size limit", + }, + ) + + // Used by scrapeCache. + sm.targetScrapeCacheFlushForced = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_cache_flush_forced_total", + Help: "How many times a scrape cache was flushed due to getting big while scrapes are failing.", + }, + ) + + // Used by scrapeLoop. + sm.targetIntervalLength = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "prometheus_target_interval_length_seconds", + Help: "Actual intervals between scrapes.", + Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, + }, + []string{"interval"}, + ) + sm.targetScrapeSampleLimit = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_exceeded_sample_limit_total", + Help: "Total number of scrapes that hit the sample limit and were rejected.", + }, + ) + sm.targetScrapeSampleDuplicate = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_sample_duplicate_timestamp_total", + Help: "Total number of samples rejected due to duplicate timestamps but different values.", + }, + ) + sm.targetScrapeSampleOutOfOrder = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_sample_out_of_order_total", + Help: "Total number of samples rejected due to not being out of the expected order.", + }, + ) + sm.targetScrapeSampleOutOfBounds = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_sample_out_of_bounds_total", + Help: "Total number of samples rejected due to timestamp falling outside of the time bounds.", + }, + ) + sm.targetScrapePoolExceededLabelLimits = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrape_pool_exceeded_label_limits_total", + Help: "Total number of times scrape pools hit the label limits, during sync or config reload.", + }, + ) + sm.targetScrapeNativeHistogramBucketLimit = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total", + Help: "Total number of scrapes that hit the native histogram bucket limit and were rejected.", + }, + ) + sm.targetScrapeExemplarOutOfOrder = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "prometheus_target_scrapes_exemplar_out_of_order_total", + Help: "Total number of exemplar rejected due to not being out of the expected order.", + }, + ) + + for _, collector := range []prometheus.Collector{ + // Used by Manager. + sm.targetMetadataCache, + sm.targetScrapePools, + sm.targetScrapePoolsFailed, + // Used by scrapePool. + sm.targetReloadIntervalLength, + sm.targetScrapePoolReloads, + sm.targetScrapePoolReloadsFailed, + sm.targetSyncIntervalLength, + sm.targetScrapePoolSyncsCounter, + sm.targetScrapePoolExceededTargetLimit, + sm.targetScrapePoolTargetLimit, + sm.targetScrapePoolTargetsAdded, + sm.targetSyncFailed, + // Used by targetScraper. + sm.targetScrapeExceededBodySizeLimit, + // Used by scrapeCache. + sm.targetScrapeCacheFlushForced, + // Used by scrapeLoop. + sm.targetIntervalLength, + sm.targetScrapeSampleLimit, + sm.targetScrapeSampleDuplicate, + sm.targetScrapeSampleOutOfOrder, + sm.targetScrapeSampleOutOfBounds, + sm.targetScrapeExemplarOutOfOrder, + sm.targetScrapePoolExceededLabelLimits, + sm.targetScrapeNativeHistogramBucketLimit, + } { + err := reg.Register(collector) + if err != nil { + return nil, fmt.Errorf("failed to register scrape metrics: %w", err) + } + } + return sm, nil +} + +func (sm *scrapeMetrics) setTargetMetadataCacheGatherer(gatherer TargetsGatherer) { + sm.targetMetadataCache.TargetsGatherer = gatherer +} + +type TargetsGatherer interface { + TargetsActive() map[string][]*Target +} + +// MetadataMetricsCollector is a Custom Collector for the metadata cache metrics. +type MetadataMetricsCollector struct { + CacheEntries *prometheus.Desc + CacheBytes *prometheus.Desc + TargetsGatherer TargetsGatherer +} + +// Describe sends the metrics descriptions to the channel. +func (mc *MetadataMetricsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- mc.CacheEntries + ch <- mc.CacheBytes +} + +// Collect creates and sends the metrics for the metadata cache. +func (mc *MetadataMetricsCollector) Collect(ch chan<- prometheus.Metric) { + if mc.TargetsGatherer == nil { + return + } + + for tset, targets := range mc.TargetsGatherer.TargetsActive() { + var size, length int + for _, t := range targets { + size += t.MetadataSize() + length += t.MetadataLength() + } + + ch <- prometheus.MustNewConstMetric( + mc.CacheEntries, + prometheus.GaugeValue, + float64(length), + tset, + ) + + ch <- prometheus.MustNewConstMetric( + mc.CacheBytes, + prometheus.GaugeValue, + float64(size), + tset, + ) + } +} diff --git a/vendor/github.com/prometheus/prometheus/scrape/scrape.go b/vendor/github.com/prometheus/prometheus/scrape/scrape.go index 5db309008c..9a0ba1d009 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/scrape.go +++ b/vendor/github.com/prometheus/prometheus/scrape/scrape.go @@ -18,19 +18,19 @@ import ( "bytes" "compress/gzip" "context" + "errors" "fmt" "io" "math" "net/http" "reflect" "strconv" + "strings" "sync" "time" "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" @@ -60,172 +60,6 @@ var AlignScrapeTimestamps = true var errNameLabelMandatory = fmt.Errorf("missing metric name (%s label)", labels.MetricName) -var ( - targetIntervalLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Name: "prometheus_target_interval_length_seconds", - Help: "Actual intervals between scrapes.", - Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, - }, - []string{"interval"}, - ) - targetReloadIntervalLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Name: "prometheus_target_reload_length_seconds", - Help: "Actual interval to reload the scrape pool with a given configuration.", - Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, - }, - []string{"interval"}, - ) - targetScrapePools = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrape_pools_total", - Help: "Total number of scrape pool creation attempts.", - }, - ) - targetScrapePoolsFailed = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrape_pools_failed_total", - Help: "Total number of scrape pool creations that failed.", - }, - ) - targetScrapePoolReloads = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrape_pool_reloads_total", - Help: "Total number of scrape pool reloads.", - }, - ) - targetScrapePoolReloadsFailed = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrape_pool_reloads_failed_total", - Help: "Total number of failed scrape pool reloads.", - }, - ) - targetScrapePoolExceededTargetLimit = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrape_pool_exceeded_target_limit_total", - Help: "Total number of times scrape pools hit the target limit, during sync or config reload.", - }, - ) - targetScrapePoolTargetLimit = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "prometheus_target_scrape_pool_target_limit", - Help: "Maximum number of targets allowed in this scrape pool.", - }, - []string{"scrape_job"}, - ) - targetScrapePoolTargetsAdded = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "prometheus_target_scrape_pool_targets", - Help: "Current number of targets in this scrape pool.", - }, - []string{"scrape_job"}, - ) - targetSyncIntervalLength = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Name: "prometheus_target_sync_length_seconds", - Help: "Actual interval to sync the scrape pool.", - Objectives: map[float64]float64{0.01: 0.001, 0.05: 0.005, 0.5: 0.05, 0.90: 0.01, 0.99: 0.001}, - }, - []string{"scrape_job"}, - ) - targetScrapePoolSyncsCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "prometheus_target_scrape_pool_sync_total", - Help: "Total number of syncs that were executed on a scrape pool.", - }, - []string{"scrape_job"}, - ) - targetScrapeExceededBodySizeLimit = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_exceeded_body_size_limit_total", - Help: "Total number of scrapes that hit the body size limit", - }, - ) - targetScrapeSampleLimit = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_exceeded_sample_limit_total", - Help: "Total number of scrapes that hit the sample limit and were rejected.", - }, - ) - targetScrapeSampleDuplicate = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_sample_duplicate_timestamp_total", - Help: "Total number of samples rejected due to duplicate timestamps but different values.", - }, - ) - targetScrapeSampleOutOfOrder = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_sample_out_of_order_total", - Help: "Total number of samples rejected due to not being out of the expected order.", - }, - ) - targetScrapeSampleOutOfBounds = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_sample_out_of_bounds_total", - Help: "Total number of samples rejected due to timestamp falling outside of the time bounds.", - }, - ) - targetScrapeCacheFlushForced = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_cache_flush_forced_total", - Help: "How many times a scrape cache was flushed due to getting big while scrapes are failing.", - }, - ) - targetScrapeExemplarOutOfOrder = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_exemplar_out_of_order_total", - Help: "Total number of exemplar rejected due to not being out of the expected order.", - }, - ) - targetScrapePoolExceededLabelLimits = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrape_pool_exceeded_label_limits_total", - Help: "Total number of times scrape pools hit the label limits, during sync or config reload.", - }, - ) - targetSyncFailed = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "prometheus_target_sync_failed_total", - Help: "Total number of target sync failures.", - }, - []string{"scrape_job"}, - ) - targetScrapeNativeHistogramBucketLimit = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total", - Help: "Total number of scrapes that hit the native histogram bucket limit and were rejected.", - }, - ) -) - -func init() { - prometheus.MustRegister( - targetIntervalLength, - targetReloadIntervalLength, - targetScrapePools, - targetScrapePoolsFailed, - targetScrapePoolReloads, - targetScrapePoolReloadsFailed, - targetSyncIntervalLength, - targetScrapePoolSyncsCounter, - targetScrapeExceededBodySizeLimit, - targetScrapeSampleLimit, - targetScrapeSampleDuplicate, - targetScrapeSampleOutOfOrder, - targetScrapeSampleOutOfBounds, - targetScrapePoolExceededTargetLimit, - targetScrapePoolTargetLimit, - targetScrapePoolTargetsAdded, - targetScrapeCacheFlushForced, - targetMetadataCache, - targetScrapeExemplarOutOfOrder, - targetScrapePoolExceededLabelLimits, - targetSyncFailed, - targetScrapeNativeHistogramBucketLimit, - ) -} - // scrapePool manages scrapes for sets of targets. type scrapePool struct { appendable storage.Appendable @@ -251,7 +85,7 @@ type scrapePool struct { noDefaultPort bool - enableProtobufNegotiation bool + metrics *scrapeMetrics } type labelLimits struct { @@ -274,45 +108,42 @@ type scrapeLoopOptions struct { scrapeClassicHistograms bool mrc []*relabel.Config cache *scrapeCache + enableCompression bool } const maxAheadTime = 10 * time.Minute -// returning an empty label set is interpreted as "drop" +// returning an empty label set is interpreted as "drop". type labelsMutator func(labels.Labels) labels.Labels -func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed uint64, logger log.Logger, options *Options) (*scrapePool, error) { - targetScrapePools.Inc() +func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed uint64, logger log.Logger, buffers *pool.Pool, options *Options, metrics *scrapeMetrics) (*scrapePool, error) { if logger == nil { logger = log.NewNopLogger() } client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, options.HTTPClientOptions...) if err != nil { - targetScrapePoolsFailed.Inc() - return nil, errors.Wrap(err, "error creating HTTP client") + return nil, fmt.Errorf("error creating HTTP client: %w", err) } - buffers := pool.New(1e3, 100e6, 3, func(sz int) interface{} { return make([]byte, 0, sz) }) - ctx, cancel := context.WithCancel(context.Background()) sp := &scrapePool{ - cancel: cancel, - appendable: app, - config: cfg, - client: client, - activeTargets: map[uint64]*Target{}, - loops: map[uint64]loop{}, - logger: logger, - httpOpts: options.HTTPClientOptions, - noDefaultPort: options.NoDefaultPort, - enableProtobufNegotiation: options.EnableProtobufNegotiation, + cancel: cancel, + appendable: app, + config: cfg, + client: client, + activeTargets: map[uint64]*Target{}, + loops: map[uint64]loop{}, + logger: logger, + metrics: metrics, + httpOpts: options.HTTPClientOptions, + noDefaultPort: options.NoDefaultPort, } sp.newLoop = func(opts scrapeLoopOptions) loop { // Update the targets retrieval function for metadata to a new scrape cache. cache := opts.cache if cache == nil { - cache = newScrapeCache() + cache = newScrapeCache(metrics) } opts.target.SetMetadataStore(cache) @@ -330,6 +161,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed offsetSeed, opts.honorTimestamps, opts.trackTimestampsStaleness, + opts.enableCompression, opts.sampleLimit, opts.bucketLimit, opts.labelLimits, @@ -340,9 +172,10 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, offsetSeed options.EnableMetadataStorage, opts.target, options.PassMetadataInContext, + metrics, ) } - targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit)) + sp.metrics.targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit)) return sp, nil } @@ -397,11 +230,11 @@ func (sp *scrapePool) stop() { sp.client.CloseIdleConnections() if sp.config != nil { - targetScrapePoolSyncsCounter.DeleteLabelValues(sp.config.JobName) - targetScrapePoolTargetLimit.DeleteLabelValues(sp.config.JobName) - targetScrapePoolTargetsAdded.DeleteLabelValues(sp.config.JobName) - targetSyncIntervalLength.DeleteLabelValues(sp.config.JobName) - targetSyncFailed.DeleteLabelValues(sp.config.JobName) + sp.metrics.targetScrapePoolSyncsCounter.DeleteLabelValues(sp.config.JobName) + sp.metrics.targetScrapePoolTargetLimit.DeleteLabelValues(sp.config.JobName) + sp.metrics.targetScrapePoolTargetsAdded.DeleteLabelValues(sp.config.JobName) + sp.metrics.targetSyncIntervalLength.DeleteLabelValues(sp.config.JobName) + sp.metrics.targetSyncFailed.DeleteLabelValues(sp.config.JobName) } } @@ -411,13 +244,13 @@ func (sp *scrapePool) stop() { func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { sp.mtx.Lock() defer sp.mtx.Unlock() - targetScrapePoolReloads.Inc() + sp.metrics.targetScrapePoolReloads.Inc() start := time.Now() client, err := config_util.NewClientFromConfig(cfg.HTTPClientConfig, cfg.JobName, sp.httpOpts...) if err != nil { - targetScrapePoolReloadsFailed.Inc() - return errors.Wrap(err, "error creating HTTP client") + sp.metrics.targetScrapePoolReloadsFailed.Inc() + return fmt.Errorf("error creating HTTP client: %w", err) } reuseCache := reusableCache(sp.config, cfg) @@ -425,7 +258,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { oldClient := sp.client sp.client = client - targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit)) + sp.metrics.targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit)) var ( wg sync.WaitGroup @@ -441,6 +274,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { } honorLabels = sp.config.HonorLabels honorTimestamps = sp.config.HonorTimestamps + enableCompression = sp.config.EnableCompression trackTimestampsStaleness = sp.config.TrackTimestampsStaleness mrc = sp.config.MetricRelabelConfigs ) @@ -454,17 +288,20 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { oldLoop.disableEndOfRunStalenessMarkers() cache = oc } else { - cache = newScrapeCache() + cache = newScrapeCache(sp.metrics) } t := sp.activeTargets[fp] interval, timeout, err := t.intervalAndTimeout(interval, timeout) - acceptHeader := scrapeAcceptHeader - if sp.enableProtobufNegotiation { - acceptHeader = scrapeAcceptHeaderWithProtobuf - } var ( - s = &targetScraper{Target: t, client: sp.client, timeout: timeout, bodySizeLimit: bodySizeLimit, acceptHeader: acceptHeader} + s = &targetScraper{ + Target: t, + client: sp.client, + timeout: timeout, + bodySizeLimit: bodySizeLimit, + acceptHeader: acceptHeader(cfg.ScrapeProtocols), + acceptEncodingHeader: acceptEncodingHeader(enableCompression), + } newLoop = sp.newLoop(scrapeLoopOptions{ target: t, scraper: s, @@ -473,6 +310,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { labelLimits: labelLimits, honorLabels: honorLabels, honorTimestamps: honorTimestamps, + enableCompression: enableCompression, trackTimestampsStaleness: trackTimestampsStaleness, mrc: mrc, cache: cache, @@ -500,7 +338,7 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { wg.Wait() oldClient.CloseIdleConnections() - targetReloadIntervalLength.WithLabelValues(interval.String()).Observe( + sp.metrics.targetReloadIntervalLength.WithLabelValues(interval.String()).Observe( time.Since(start).Seconds(), ) return nil @@ -524,7 +362,7 @@ func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { for _, err := range failures { level.Error(sp.logger).Log("msg", "Creating target failed", "err", err) } - targetSyncFailed.WithLabelValues(sp.config.JobName).Add(float64(len(failures))) + sp.metrics.targetSyncFailed.WithLabelValues(sp.config.JobName).Add(float64(len(failures))) for _, t := range targets { // Replicate .Labels().IsEmpty() with a loop here to avoid generating garbage. nonEmpty := false @@ -543,10 +381,10 @@ func (sp *scrapePool) Sync(tgs []*targetgroup.Group) { sp.targetMtx.Unlock() sp.sync(all) - targetSyncIntervalLength.WithLabelValues(sp.config.JobName).Observe( + sp.metrics.targetSyncIntervalLength.WithLabelValues(sp.config.JobName).Observe( time.Since(start).Seconds(), ) - targetScrapePoolSyncsCounter.WithLabelValues(sp.config.JobName).Inc() + sp.metrics.targetScrapePoolSyncsCounter.WithLabelValues(sp.config.JobName).Inc() } // sync takes a list of potentially duplicated targets, deduplicates them, starts @@ -567,6 +405,7 @@ func (sp *scrapePool) sync(targets []*Target) { } honorLabels = sp.config.HonorLabels honorTimestamps = sp.config.HonorTimestamps + enableCompression = sp.config.EnableCompression trackTimestampsStaleness = sp.config.TrackTimestampsStaleness mrc = sp.config.MetricRelabelConfigs scrapeClassicHistograms = sp.config.ScrapeClassicHistograms @@ -582,11 +421,15 @@ func (sp *scrapePool) sync(targets []*Target) { // for every target. var err error interval, timeout, err = t.intervalAndTimeout(interval, timeout) - acceptHeader := scrapeAcceptHeader - if sp.enableProtobufNegotiation { - acceptHeader = scrapeAcceptHeaderWithProtobuf + s := &targetScraper{ + Target: t, + client: sp.client, + timeout: timeout, + bodySizeLimit: bodySizeLimit, + acceptHeader: acceptHeader(sp.config.ScrapeProtocols), + acceptEncodingHeader: acceptEncodingHeader(enableCompression), + metrics: sp.metrics, } - s := &targetScraper{Target: t, client: sp.client, timeout: timeout, bodySizeLimit: bodySizeLimit, acceptHeader: acceptHeader} l := sp.newLoop(scrapeLoopOptions{ target: t, scraper: s, @@ -595,6 +438,7 @@ func (sp *scrapePool) sync(targets []*Target) { labelLimits: labelLimits, honorLabels: honorLabels, honorTimestamps: honorTimestamps, + enableCompression: enableCompression, trackTimestampsStaleness: trackTimestampsStaleness, mrc: mrc, interval: interval, @@ -638,7 +482,7 @@ func (sp *scrapePool) sync(targets []*Target) { sp.targetMtx.Unlock() - targetScrapePoolTargetsAdded.WithLabelValues(sp.config.JobName).Set(float64(len(uniqueLoops))) + sp.metrics.targetScrapePoolTargetsAdded.WithLabelValues(sp.config.JobName).Set(float64(len(uniqueLoops))) forcedErr := sp.refreshTargetLimitErr() for _, l := range sp.loops { l.setForcedError(forcedErr) @@ -662,7 +506,7 @@ func (sp *scrapePool) refreshTargetLimitErr() error { return nil } if l := len(sp.activeTargets); l > int(sp.config.TargetLimit) { - targetScrapePoolExceededTargetLimit.Inc() + sp.metrics.targetScrapePoolExceededTargetLimit.Inc() return fmt.Errorf("target_limit exceeded (number of targets: %d, limit: %d)", l, sp.config.TargetLimit) } return nil @@ -808,16 +652,36 @@ type targetScraper struct { gzipr *gzip.Reader buf *bufio.Reader - bodySizeLimit int64 - acceptHeader string + bodySizeLimit int64 + acceptHeader string + acceptEncodingHeader string + + metrics *scrapeMetrics } var errBodySizeLimit = errors.New("body size limit exceeded") -const ( - scrapeAcceptHeader = `application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1;q=0.75,text/plain;version=0.0.4;q=0.5,*/*;q=0.1` - scrapeAcceptHeaderWithProtobuf = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited,application/openmetrics-text;version=1.0.0;q=0.8,application/openmetrics-text;version=0.0.1;q=0.75,text/plain;version=0.0.4;q=0.5,*/*;q=0.1` -) +// acceptHeader transforms preference from the options into specific header values as +// https://www.rfc-editor.org/rfc/rfc9110.html#name-accept defines. +// No validation is here, we expect scrape protocols to be validated already. +func acceptHeader(sps []config.ScrapeProtocol) string { + var vals []string + weight := len(config.ScrapeProtocolsHeaders) + 1 + for _, sp := range sps { + vals = append(vals, fmt.Sprintf("%s;q=0.%d", config.ScrapeProtocolsHeaders[sp], weight)) + weight-- + } + // Default match anything. + vals = append(vals, fmt.Sprintf("*/*;q=%d", weight)) + return strings.Join(vals, ",") +} + +func acceptEncodingHeader(enableCompression bool) string { + if enableCompression { + return "gzip" + } + return "identity" +} var UserAgent = fmt.Sprintf("Prometheus/%s", version.Version) @@ -828,7 +692,7 @@ func (s *targetScraper) scrape(ctx context.Context) (*http.Response, error) { return nil, err } req.Header.Add("Accept", s.acceptHeader) - req.Header.Add("Accept-Encoding", "gzip") + req.Header.Add("Accept-Encoding", s.acceptEncodingHeader) req.Header.Set("User-Agent", UserAgent) req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", strconv.FormatFloat(s.timeout.Seconds(), 'f', -1, 64)) @@ -845,7 +709,7 @@ func (s *targetScraper) readResponse(ctx context.Context, resp *http.Response, w }() if resp.StatusCode != http.StatusOK { - return "", errors.Errorf("server returned HTTP status %s", resp.Status) + return "", fmt.Errorf("server returned HTTP status %s", resp.Status) } if s.bodySizeLimit <= 0 { @@ -857,7 +721,7 @@ func (s *targetScraper) readResponse(ctx context.Context, resp *http.Response, w return "", err } if n >= s.bodySizeLimit { - targetScrapeExceededBodySizeLimit.Inc() + s.metrics.targetScrapeExceededBodySizeLimit.Inc() return "", errBodySizeLimit } return resp.Header.Get("Content-Type"), nil @@ -883,7 +747,7 @@ func (s *targetScraper) readResponse(ctx context.Context, resp *http.Response, w return "", err } if n >= s.bodySizeLimit { - targetScrapeExceededBodySizeLimit.Inc() + s.metrics.targetScrapeExceededBodySizeLimit.Inc() return "", errBodySizeLimit } return resp.Header.Get("Content-Type"), nil @@ -914,6 +778,7 @@ type scrapeLoop struct { offsetSeed uint64 honorTimestamps bool trackTimestampsStaleness bool + enableCompression bool forcedErr error forcedErrMtx sync.Mutex sampleLimit int @@ -937,6 +802,8 @@ type scrapeLoop struct { reportExtraMetrics bool appendMetadataToWAL bool + + metrics *scrapeMetrics } // scrapeCache tracks mappings of exposed metric strings to label sets and @@ -964,6 +831,8 @@ type scrapeCache struct { metaMtx sync.Mutex metadata map[string]*metaEntry + + metrics *scrapeMetrics } // metaEntry holds meta information about a metric. @@ -979,13 +848,14 @@ func (m *metaEntry) size() int { return len(m.Help) + len(m.Unit) + len(m.Type) } -func newScrapeCache() *scrapeCache { +func newScrapeCache(metrics *scrapeMetrics) *scrapeCache { return &scrapeCache{ series: map[string]*cacheEntry{}, droppedSeries: map[string]*uint64{}, seriesCur: map[uint64]labels.Labels{}, seriesPrev: map[uint64]labels.Labels{}, metadata: map[string]*metaEntry{}, + metrics: metrics, } } @@ -1004,7 +874,7 @@ func (c *scrapeCache) iterDone(flushCache bool) { // since the last scrape, and allow an additional 1000 in case // initial scrapes all fail. flushCache = true - targetScrapeCacheFlushForced.Inc() + c.metrics.targetScrapeCacheFlushForced.Inc() } if flushCache { @@ -1199,6 +1069,7 @@ func newScrapeLoop(ctx context.Context, offsetSeed uint64, honorTimestamps bool, trackTimestampsStaleness bool, + enableCompression bool, sampleLimit int, bucketLimit int, labelLimits *labelLimits, @@ -1209,6 +1080,7 @@ func newScrapeLoop(ctx context.Context, appendMetadataToWAL bool, target *Target, passMetadataInContext bool, + metrics *scrapeMetrics, ) *scrapeLoop { if l == nil { l = log.NewNopLogger() @@ -1217,7 +1089,7 @@ func newScrapeLoop(ctx context.Context, buffers = pool.New(1e3, 1e6, 3, func(sz int) interface{} { return make([]byte, 0, sz) }) } if cache == nil { - cache = newScrapeCache() + cache = newScrapeCache(metrics) } appenderCtx := ctx @@ -1245,6 +1117,7 @@ func newScrapeLoop(ctx context.Context, appenderCtx: appenderCtx, honorTimestamps: honorTimestamps, trackTimestampsStaleness: trackTimestampsStaleness, + enableCompression: enableCompression, sampleLimit: sampleLimit, bucketLimit: bucketLimit, labelLimits: labelLimits, @@ -1253,6 +1126,7 @@ func newScrapeLoop(ctx context.Context, scrapeClassicHistograms: scrapeClassicHistograms, reportExtraMetrics: reportExtraMetrics, appendMetadataToWAL: appendMetadataToWAL, + metrics: metrics, } sl.ctx, sl.cancel = context.WithCancel(ctx) @@ -1332,7 +1206,7 @@ func (sl *scrapeLoop) scrapeAndReport(last, appendTime time.Time, errc chan<- er // Only record after the first scrape. if !last.IsZero() { - targetIntervalLength.WithLabelValues(sl.interval.String()).Observe( + sl.metrics.targetIntervalLength.WithLabelValues(sl.interval.String()).Observe( time.Since(last).Seconds(), ) } @@ -1547,6 +1421,8 @@ func (sl *scrapeLoop) append(app storage.Appender, b []byte, contentType string, metadataChanged bool ) + exemplars := make([]exemplar.Exemplar, 1) + // updateMetadata updates the current iteration's metadata object and the // metadataChanged value if we have metadata in the scrape cache AND the // labelset is for a new series or the metadata for this series has just @@ -1673,7 +1549,7 @@ loop: // If any label limits is exceeded the scrape should fail. if err = verifyLabelLimits(lset, sl.labelLimits); err != nil { - targetScrapePoolExceededLabelLimits.Inc() + sl.metrics.targetScrapePoolExceededLabelLimits.Inc() break loop } @@ -1692,7 +1568,7 @@ loop: } sampleAdded, err = sl.checkAddError(ce, met, parsedTimestamp, err, &sampleLimitErr, &bucketLimitErr, &appErrs) if err != nil { - if err != storage.ErrNotFound { + if !errors.Is(err, storage.ErrNotFound) { level.Debug(sl.l).Log("msg", "Unexpected error", "series", string(met), "err", err) } break loop @@ -1712,18 +1588,46 @@ loop: // Increment added even if there's an error so we correctly report the // number of samples remaining after relabeling. added++ - + exemplars = exemplars[:0] // Reset and reuse the exemplar slice. for hasExemplar := p.Exemplar(&e); hasExemplar; hasExemplar = p.Exemplar(&e) { if !e.HasTs { + if isHistogram { + // We drop exemplars for native histograms if they don't have a timestamp. + // Missing timestamps are deliberately not supported as we want to start + // enforcing timestamps for exemplars as otherwise proper deduplication + // is inefficient and purely based on heuristics: we cannot distinguish + // between repeated exemplars and new instances with the same values. + // This is done silently without logs as it is not an error but out of spec. + // This does not affect classic histograms so that behaviour is unchanged. + e = exemplar.Exemplar{} // Reset for next time round loop. + continue + } e.Ts = t } + exemplars = append(exemplars, e) + e = exemplar.Exemplar{} // Reset for next time round loop. + } + // Sort so that checking for duplicates / out of order is more efficient during validation. + slices.SortFunc(exemplars, exemplar.Compare) + outOfOrderExemplars := 0 + for _, e := range exemplars { _, exemplarErr := app.AppendExemplar(ref, lset, e) - exemplarErr = sl.checkAddExemplarError(exemplarErr, e, &appErrs) - if exemplarErr != nil { + switch { + case exemplarErr == nil: + // Do nothing. + case errors.Is(exemplarErr, storage.ErrOutOfOrderExemplar): + outOfOrderExemplars++ + default: // Since exemplar storage is still experimental, we don't fail the scrape on ingestion errors. level.Debug(sl.l).Log("msg", "Error while adding exemplar in AddExemplar", "exemplar", fmt.Sprintf("%+v", e), "err", exemplarErr) } - e = exemplar.Exemplar{} // reset for next time round loop + } + if outOfOrderExemplars > 0 && outOfOrderExemplars == len(exemplars) { + // Only report out of order exemplars if all are out of order, otherwise this was a partial update + // to some existing set of exemplars. + appErrs.numExemplarOutOfOrder += outOfOrderExemplars + level.Debug(sl.l).Log("msg", "Out of order exemplars", "count", outOfOrderExemplars, "latest", fmt.Sprintf("%+v", exemplars[len(exemplars)-1])) + sl.metrics.targetScrapeExemplarOutOfOrder.Add(float64(outOfOrderExemplars)) } if sl.appendMetadataToWAL && metadataChanged { @@ -1738,14 +1642,14 @@ loop: err = sampleLimitErr } // We only want to increment this once per scrape, so this is Inc'd outside the loop. - targetScrapeSampleLimit.Inc() + sl.metrics.targetScrapeSampleLimit.Inc() } if bucketLimitErr != nil { if err == nil { err = bucketLimitErr // If sample limit is hit, that error takes precedence. } // We only want to increment this once per scrape, so this is Inc'd outside the loop. - targetScrapeNativeHistogramBucketLimit.Inc() + sl.metrics.targetScrapeNativeHistogramBucketLimit.Inc() } if appErrs.numOutOfOrder > 0 { level.Warn(sl.l).Log("msg", "Error on ingesting out-of-order samples", "num_dropped", appErrs.numOutOfOrder) @@ -1763,8 +1667,8 @@ loop: sl.cache.forEachStale(func(lset labels.Labels) bool { // Series no longer exposed, mark it stale. _, err = app.Append(0, lset, defTime, math.Float64frombits(value.StaleNaN)) - switch errors.Cause(err) { - case storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp: + switch { + case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): // Do not count these in logging, as this is expected if a target // goes away and comes back again with a new scrape loop. err = nil @@ -1778,35 +1682,35 @@ loop: // Adds samples to the appender, checking the error, and then returns the # of samples added, // whether the caller should continue to process more samples, and any sample or bucket limit errors. func (sl *scrapeLoop) checkAddError(ce *cacheEntry, met []byte, tp *int64, err error, sampleLimitErr, bucketLimitErr *error, appErrs *appendErrors) (bool, error) { - switch errors.Cause(err) { - case nil: + switch { + case err == nil: if (tp == nil || sl.trackTimestampsStaleness) && ce != nil { sl.cache.trackStaleness(ce.hash, ce.lset) } return true, nil - case storage.ErrNotFound: + case errors.Is(err, storage.ErrNotFound): return false, storage.ErrNotFound - case storage.ErrOutOfOrderSample: + case errors.Is(err, storage.ErrOutOfOrderSample): appErrs.numOutOfOrder++ level.Debug(sl.l).Log("msg", "Out of order sample", "series", string(met)) - targetScrapeSampleOutOfOrder.Inc() + sl.metrics.targetScrapeSampleOutOfOrder.Inc() return false, nil - case storage.ErrDuplicateSampleForTimestamp: + case errors.Is(err, storage.ErrDuplicateSampleForTimestamp): appErrs.numDuplicates++ level.Debug(sl.l).Log("msg", "Duplicate sample for timestamp", "series", string(met)) - targetScrapeSampleDuplicate.Inc() + sl.metrics.targetScrapeSampleDuplicate.Inc() return false, nil - case storage.ErrOutOfBounds: + case errors.Is(err, storage.ErrOutOfBounds): appErrs.numOutOfBounds++ level.Debug(sl.l).Log("msg", "Out of bounds metric", "series", string(met)) - targetScrapeSampleOutOfBounds.Inc() + sl.metrics.targetScrapeSampleOutOfBounds.Inc() return false, nil - case errSampleLimit: + case errors.Is(err, errSampleLimit): // Keep on parsing output if we hit the limit, so we report the correct // total number of samples scraped. *sampleLimitErr = err return false, nil - case errBucketLimit: + case errors.Is(err, errBucketLimit): // Keep on parsing output if we hit the limit, so we report the correct // total number of samples scraped. *bucketLimitErr = err @@ -1816,20 +1720,6 @@ func (sl *scrapeLoop) checkAddError(ce *cacheEntry, met []byte, tp *int64, err e } } -func (sl *scrapeLoop) checkAddExemplarError(err error, e exemplar.Exemplar, appErrs *appendErrors) error { - switch errors.Cause(err) { - case storage.ErrNotFound: - return storage.ErrNotFound - case storage.ErrOutOfOrderExemplar: - appErrs.numExemplarOutOfOrder++ - level.Debug(sl.l).Log("msg", "Out of order exemplar", "exemplar", fmt.Sprintf("%+v", e)) - targetScrapeExemplarOutOfOrder.Inc() - return nil - default: - return err - } -} - // The constants are suffixed with the invalid \xff unicode rune to avoid collisions // with scraped metrics in the cache. var ( @@ -1932,13 +1822,13 @@ func (sl *scrapeLoop) addReportSample(app storage.Appender, s []byte, t int64, v } ref, err := app.Append(ref, lset, t, v) - switch errors.Cause(err) { - case nil: + switch { + case err == nil: if !ok { sl.cache.addRef(s, ref, lset, lset.Hash()) } return nil - case storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp: + case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): // Do not log here, as this is expected if a target goes away and comes back // again with a new scrape loop. return nil diff --git a/vendor/github.com/prometheus/prometheus/scrape/target.go b/vendor/github.com/prometheus/prometheus/scrape/target.go index 8b745a9c49..62f662693b 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/target.go +++ b/vendor/github.com/prometheus/prometheus/scrape/target.go @@ -14,6 +14,7 @@ package scrape import ( + "errors" "fmt" "hash/fnv" "net" @@ -22,7 +23,6 @@ import ( "sync" "time" - "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" @@ -145,9 +145,7 @@ func (t *Target) SetMetadataStore(s MetricMetadataStore) { func (t *Target) hash() uint64 { h := fnv.New64a() - //nolint: errcheck h.Write([]byte(fmt.Sprintf("%016d", t.labels.Hash()))) - //nolint: errcheck h.Write([]byte(t.URL().String())) return h.Sum64() @@ -198,7 +196,7 @@ func (t *Target) DiscoveredLabels() labels.Labels { return t.discoveredLabels.Copy() } -// SetDiscoveredLabels sets new DiscoveredLabels +// SetDiscoveredLabels sets new DiscoveredLabels. func (t *Target) SetDiscoveredLabels(l labels.Labels) { t.mtx.Lock() defer t.mtx.Unlock() @@ -291,12 +289,12 @@ func (t *Target) intervalAndTimeout(defaultInterval, defaultDuration time.Durati intervalLabel := t.labels.Get(model.ScrapeIntervalLabel) interval, err := model.ParseDuration(intervalLabel) if err != nil { - return defaultInterval, defaultDuration, errors.Errorf("Error parsing interval label %q: %v", intervalLabel, err) + return defaultInterval, defaultDuration, fmt.Errorf("Error parsing interval label %q: %w", intervalLabel, err) } timeoutLabel := t.labels.Get(model.ScrapeTimeoutLabel) timeout, err := model.ParseDuration(timeoutLabel) if err != nil { - return defaultInterval, defaultDuration, errors.Errorf("Error parsing timeout label %q: %v", timeoutLabel, err) + return defaultInterval, defaultDuration, fmt.Errorf("Error parsing timeout label %q: %w", timeoutLabel, err) } return time.Duration(interval), time.Duration(timeout), nil @@ -368,13 +366,19 @@ type bucketLimitAppender struct { func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { if h != nil { - if len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit { - return 0, errBucketLimit + for len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit { + if h.Schema == -4 { + return 0, errBucketLimit + } + h = h.ReduceResolution(h.Schema - 1) } } if fh != nil { - if len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit { - return 0, errBucketLimit + for len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit { + if fh.Schema == -4 { + return 0, errBucketLimit + } + fh = fh.ReduceResolution(fh.Schema - 1) } } ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh) @@ -446,7 +450,7 @@ func PopulateLabels(lb *labels.Builder, cfg *config.ScrapeConfig, noDefaultPort case "https": addr += ":443" default: - return labels.EmptyLabels(), labels.EmptyLabels(), errors.Errorf("invalid scheme: %q", cfg.Scheme) + return labels.EmptyLabels(), labels.EmptyLabels(), fmt.Errorf("invalid scheme: %q", cfg.Scheme) } lb.Set(model.AddressLabel, addr) } @@ -473,7 +477,7 @@ func PopulateLabels(lb *labels.Builder, cfg *config.ScrapeConfig, noDefaultPort interval := lb.Get(model.ScrapeIntervalLabel) intervalDuration, err := model.ParseDuration(interval) if err != nil { - return labels.EmptyLabels(), labels.EmptyLabels(), errors.Errorf("error parsing scrape interval: %v", err) + return labels.EmptyLabels(), labels.EmptyLabels(), fmt.Errorf("error parsing scrape interval: %w", err) } if time.Duration(intervalDuration) == 0 { return labels.EmptyLabels(), labels.EmptyLabels(), errors.New("scrape interval cannot be 0") @@ -482,14 +486,14 @@ func PopulateLabels(lb *labels.Builder, cfg *config.ScrapeConfig, noDefaultPort timeout := lb.Get(model.ScrapeTimeoutLabel) timeoutDuration, err := model.ParseDuration(timeout) if err != nil { - return labels.EmptyLabels(), labels.EmptyLabels(), errors.Errorf("error parsing scrape timeout: %v", err) + return labels.EmptyLabels(), labels.EmptyLabels(), fmt.Errorf("error parsing scrape timeout: %w", err) } if time.Duration(timeoutDuration) == 0 { return labels.EmptyLabels(), labels.EmptyLabels(), errors.New("scrape timeout cannot be 0") } if timeoutDuration > intervalDuration { - return labels.EmptyLabels(), labels.EmptyLabels(), errors.Errorf("scrape timeout cannot be greater than scrape interval (%q > %q)", timeout, interval) + return labels.EmptyLabels(), labels.EmptyLabels(), fmt.Errorf("scrape timeout cannot be greater than scrape interval (%q > %q)", timeout, interval) } // Meta labels are deleted after relabelling. Other internal labels propagate to @@ -509,7 +513,7 @@ func PopulateLabels(lb *labels.Builder, cfg *config.ScrapeConfig, noDefaultPort err = res.Validate(func(l labels.Label) error { // Check label values are valid, drop the target if not. if !model.LabelValue(l.Value).IsValid() { - return errors.Errorf("invalid label value for %q: %q", l.Name, l.Value) + return fmt.Errorf("invalid label value for %q: %q", l.Name, l.Value) } return nil }) @@ -538,7 +542,7 @@ func TargetsFromGroup(tg *targetgroup.Group, cfg *config.ScrapeConfig, noDefault lset, origLabels, err := PopulateLabels(lb, cfg, noDefaultPort) if err != nil { - failures = append(failures, errors.Wrapf(err, "instance %d in group %s", i, tg)) + failures = append(failures, fmt.Errorf("instance %d in group %s: %w", i, tg, err)) } if !lset.IsEmpty() || !origLabels.IsEmpty() { targets = append(targets, NewTarget(lset, origLabels, cfg.Params)) diff --git a/vendor/github.com/prometheus/prometheus/storage/interface.go b/vendor/github.com/prometheus/prometheus/storage/interface.go index 79ca658eb3..2b1b6a63eb 100644 --- a/vendor/github.com/prometheus/prometheus/storage/interface.go +++ b/vendor/github.com/prometheus/prometheus/storage/interface.go @@ -37,16 +37,12 @@ var ( // ErrTooOldSample is when out of order support is enabled but the sample is outside the time window allowed. ErrTooOldSample = errors.New("too old sample") // ErrDuplicateSampleForTimestamp is when the sample has same timestamp but different value. - ErrDuplicateSampleForTimestamp = errors.New("duplicate sample for timestamp") - ErrOutOfOrderExemplar = errors.New("out of order exemplar") - ErrDuplicateExemplar = errors.New("duplicate exemplar") - ErrExemplarLabelLength = fmt.Errorf("label length for exemplar exceeds maximum of %d UTF-8 characters", exemplar.ExemplarMaxLabelSetLength) - ErrExemplarsDisabled = fmt.Errorf("exemplar storage is disabled or max exemplars is less than or equal to 0") - ErrNativeHistogramsDisabled = fmt.Errorf("native histograms are disabled") - ErrHistogramCountNotBigEnough = errors.New("histogram's observation count should be at least the number of observations found in the buckets") - ErrHistogramNegativeBucketCount = errors.New("histogram has a bucket whose observation count is negative") - ErrHistogramSpanNegativeOffset = errors.New("histogram has a span whose offset is negative") - ErrHistogramSpansBucketsMismatch = errors.New("histogram spans specify different number of buckets than provided") + ErrDuplicateSampleForTimestamp = errors.New("duplicate sample for timestamp") + ErrOutOfOrderExemplar = errors.New("out of order exemplar") + ErrDuplicateExemplar = errors.New("duplicate exemplar") + ErrExemplarLabelLength = fmt.Errorf("label length for exemplar exceeds maximum of %d UTF-8 characters", exemplar.ExemplarMaxLabelSetLength) + ErrExemplarsDisabled = fmt.Errorf("exemplar storage is disabled or max exemplars is less than or equal to 0") + ErrNativeHistogramsDisabled = fmt.Errorf("native histograms are disabled") ) // SeriesRef is a generic series reference. In prometheus it is either a @@ -327,7 +323,7 @@ func (s testSeriesSet) At() Series { return s.series } func (s testSeriesSet) Err() error { return nil } func (s testSeriesSet) Warnings() annotations.Annotations { return nil } -// TestSeriesSet returns a mock series set +// TestSeriesSet returns a mock series set. func TestSeriesSet(series Series) SeriesSet { return testSeriesSet{series: series} } diff --git a/vendor/github.com/prometheus/prometheus/storage/merge.go b/vendor/github.com/prometheus/prometheus/storage/merge.go index 1e29374e53..b4ebb440f3 100644 --- a/vendor/github.com/prometheus/prometheus/storage/merge.go +++ b/vendor/github.com/prometheus/prometheus/storage/merge.go @@ -473,10 +473,10 @@ func ChainSampleIteratorFromSeries(it chunkenc.Iterator, series []Series) chunke return csi } -func ChainSampleIteratorFromMetas(it chunkenc.Iterator, chunks []chunks.Meta) chunkenc.Iterator { - csi := getChainSampleIterator(it, len(chunks)) - for i, c := range chunks { - csi.iterators[i] = c.Chunk.Iterator(csi.iterators[i]) +func ChainSampleIteratorFromIterables(it chunkenc.Iterator, iterables []chunkenc.Iterable) chunkenc.Iterator { + csi := getChainSampleIterator(it, len(iterables)) + for i, c := range iterables { + csi.iterators[i] = c.Iterator(csi.iterators[i]) } return csi } @@ -497,9 +497,14 @@ func (c *chainSampleIterator) Seek(t int64) chunkenc.ValueType { c.consecutive = false c.h = samplesIteratorHeap{} for _, iter := range c.iterators { - if iter.Seek(t) != chunkenc.ValNone { - heap.Push(&c.h, iter) + if iter.Seek(t) == chunkenc.ValNone { + if iter.Err() != nil { + // If any iterator is reporting an error, abort. + return chunkenc.ValNone + } + continue } + heap.Push(&c.h, iter) } if len(c.h) > 0 { c.curr = heap.Pop(&c.h).(chunkenc.Iterator) @@ -571,7 +576,13 @@ func (c *chainSampleIterator) Next() chunkenc.ValueType { // So, we don't call Next() on it here. c.curr = c.iterators[0] for _, iter := range c.iterators[1:] { - if iter.Next() != chunkenc.ValNone { + if iter.Next() == chunkenc.ValNone { + if iter.Err() != nil { + // If any iterator is reporting an error, abort. + // If c.iterators[0] is reporting an error, we'll handle that below. + return chunkenc.ValNone + } + } else { heap.Push(&c.h, iter) } } @@ -583,7 +594,19 @@ func (c *chainSampleIterator) Next() chunkenc.ValueType { for { currValueType = c.curr.Next() - if currValueType != chunkenc.ValNone { + + if currValueType == chunkenc.ValNone { + if c.curr.Err() != nil { + // Abort if we've hit an error. + return chunkenc.ValNone + } + + if len(c.h) == 0 { + // No iterator left to iterate. + c.curr = nil + return chunkenc.ValNone + } + } else { currT = c.curr.AtT() if currT == c.lastT { // Ignoring sample for the same timestamp. @@ -603,10 +626,6 @@ func (c *chainSampleIterator) Next() chunkenc.ValueType { } // Current iterator does not hold the smallest timestamp. heap.Push(&c.h, c.curr) - } else if len(c.h) == 0 { - // No iterator left to iterate. - c.curr = nil - return chunkenc.ValNone } c.curr = heap.Pop(&c.h).(chunkenc.Iterator) @@ -841,6 +860,9 @@ func (c *concatenatingChunkIterator) Next() bool { c.curr = c.iterators[c.idx].At() return true } + if c.iterators[c.idx].Err() != nil { + return false + } c.idx++ return c.Next() } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/azuread/azuread.go b/vendor/github.com/prometheus/prometheus/storage/remote/azuread/azuread.go index cb4587b02a..20d48d0087 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/azuread/azuread.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/azuread/azuread.go @@ -43,7 +43,7 @@ const ( IngestionPublicAudience = "https://monitor.azure.com//.default" ) -// ManagedIdentityConfig is used to store managed identity config values +// ManagedIdentityConfig is used to store managed identity config values. type ManagedIdentityConfig struct { // ClientID is the clientId of the managed identity that is being used to authenticate. ClientID string `yaml:"client_id,omitempty"` @@ -62,7 +62,7 @@ type OAuthConfig struct { } // AzureADConfig is used to store the config values. -type AzureADConfig struct { // nolint:revive +type AzureADConfig struct { //nolint:revive // exported. // ManagedIdentity is the managed identity that is being used to authenticate. ManagedIdentity *ManagedIdentityConfig `yaml:"managed_identity,omitempty"` @@ -235,7 +235,7 @@ func newManagedIdentityTokenCredential(clientOpts *azcore.ClientOptions, managed return azidentity.NewManagedIdentityCredential(opts) } -// newOAuthTokenCredential returns new OAuth token credential +// newOAuthTokenCredential returns new OAuth token credential. func newOAuthTokenCredential(clientOpts *azcore.ClientOptions, oAuthConfig *OAuthConfig) (azcore.TokenCredential, error) { opts := &azidentity.ClientSecretCredentialOptions{ClientOptions: *clientOpts} return azidentity.NewClientSecretCredential(oAuthConfig.TenantID, oAuthConfig.ClientID, oAuthConfig.ClientSecret, opts) @@ -326,7 +326,7 @@ func getAudience(cloud string) (string, error) { } } -// getCloudConfiguration returns the cloud Configuration which contains AAD endpoint for different clouds +// getCloudConfiguration returns the cloud Configuration which contains AAD endpoint for different clouds. func getCloudConfiguration(c string) (cloud.Configuration, error) { switch strings.ToLower(c) { case strings.ToLower(AzureChina): diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/codec.go b/vendor/github.com/prometheus/prometheus/storage/remote/codec.go index 4c190f2a4e..67035cd8ec 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/codec.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/codec.go @@ -475,7 +475,7 @@ func (c *concreteSeriesIterator) At() (t int64, v float64) { return s.Timestamp, s.Value } -// AtHistogram implements chunkenc.Iterator +// AtHistogram implements chunkenc.Iterator. func (c *concreteSeriesIterator) AtHistogram() (int64, *histogram.Histogram) { if c.curValType != chunkenc.ValHistogram { panic("iterator is not on an integer histogram sample") @@ -484,7 +484,7 @@ func (c *concreteSeriesIterator) AtHistogram() (int64, *histogram.Histogram) { return h.Timestamp, HistogramProtoToHistogram(h) } -// AtFloatHistogram implements chunkenc.Iterator +// AtFloatHistogram implements chunkenc.Iterator. func (c *concreteSeriesIterator) AtFloatHistogram() (int64, *histogram.FloatHistogram) { switch c.curValType { case chunkenc.ValHistogram: @@ -547,7 +547,7 @@ func (c *concreteSeriesIterator) Err() error { } // validateLabelsAndMetricName validates the label names/values and metric names returned from remote read, -// also making sure that there are no labels with duplicate names +// also making sure that there are no labels with duplicate names. func validateLabelsAndMetricName(ls []prompb.Label) error { for i, l := range ls { if l.Name == labels.MetricName && !model.IsValidMetricName(model.LabelValue(l.Value)) { @@ -752,7 +752,7 @@ func spansToSpansProto(s []histogram.Span) []prompb.BucketSpan { return spans } -// LabelProtosToMetric unpack a []*prompb.Label to a model.Metric +// LabelProtosToMetric unpack a []*prompb.Label to a model.Metric. func LabelProtosToMetric(labelPairs []*prompb.Label) model.Metric { metric := make(model.Metric, len(labelPairs)) for _, l := range labelPairs { diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go index cfcb5652ee..b44ba869fe 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_label.go @@ -1,11 +1,20 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package normalize +package prometheus // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus" import ( "strings" "unicode" + + "go.opentelemetry.io/collector/featuregate" +) + +var dropSanitizationGate = featuregate.GlobalRegistry().MustRegister( + "pkg.translator.prometheus.PermissiveLabelSanitization", + featuregate.StageAlpha, + featuregate.WithRegisterDescription("Controls whether to change labels starting with '_' to 'key_'."), + featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/8950"), ) // Normalizes the specified label to follow Prometheus label names standard @@ -27,6 +36,8 @@ func NormalizeLabel(label string) string { // If label starts with a number, prepend with "key_" if unicode.IsDigit(rune(label[0])) { label = "key_" + label + } else if strings.HasPrefix(label, "_") && !strings.HasPrefix(label, "__") && !dropSanitizationGate.IsEnabled() { + label = "key" + label } return label diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go index 0b62a28f24..72fc04cea2 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/normalize_name.go @@ -1,12 +1,13 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package normalize +package prometheus // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus" import ( "strings" "unicode" + "go.opentelemetry.io/collector/featuregate" "go.opentelemetry.io/collector/pdata/pmetric" ) @@ -16,6 +17,7 @@ import ( // Prometheus best practices for units: https://prometheus.io/docs/practices/naming/#base-units // OpenMetrics specification for units: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#units-and-base-units var unitMap = map[string]string{ + // Time "d": "days", "h": "hours", @@ -35,11 +37,6 @@ var unitMap = map[string]string{ "MBy": "megabytes", "GBy": "gigabytes", "TBy": "terabytes", - "B": "bytes", - "KB": "kilobytes", - "MB": "megabytes", - "GB": "gigabytes", - "TB": "terabytes", // SI "m": "meters", @@ -54,7 +51,6 @@ var unitMap = map[string]string{ "Hz": "hertz", "1": "", "%": "percent", - "$": "dollars", } // The map that translates the "per" unit @@ -69,7 +65,14 @@ var perUnitMap = map[string]string{ "y": "year", } -// Build a Prometheus-compliant metric name for the specified metric +var normalizeNameGate = featuregate.GlobalRegistry().MustRegister( + "pkg.translator.prometheus.NormalizeName", + featuregate.StageBeta, + featuregate.WithRegisterDescription("Controls whether metrics names are automatically normalized to follow Prometheus naming convention"), + featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/8950"), +) + +// BuildCompliantName builds a Prometheus-compliant metric name for the specified metric // // Metric name is prefixed with specified namespace and underscore (if any). // Namespace is not cleaned up. Make sure specified namespace follows Prometheus @@ -77,7 +80,33 @@ var perUnitMap = map[string]string{ // // See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels // and https://prometheus.io/docs/practices/naming/#metric-and-label-naming -func BuildPromCompliantName(metric pmetric.Metric, namespace string) string { +func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffixes bool) string { + var metricName string + + // Full normalization following standard Prometheus naming conventions + if addMetricSuffixes && normalizeNameGate.IsEnabled() { + return normalizeName(metric, namespace) + } + + // Simple case (no full normalization, no units, etc.), we simply trim out forbidden chars + metricName = RemovePromForbiddenRunes(metric.Name()) + + // Namespace? + if namespace != "" { + return namespace + "_" + metricName + } + + // Metric name starts with a digit? Prefix it with an underscore + if metricName != "" && unicode.IsDigit(rune(metricName[0])) { + metricName = "_" + metricName + } + + return metricName +} + +// Build a normalized name for the specified metric +func normalizeName(metric pmetric.Metric, namespace string) string { + // Split metric name in "tokens" (remove all non-alphanumeric) nameTokens := strings.FieldsFunc( metric.Name(), diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/unit_to_ucum.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/unit_to_ucum.go new file mode 100644 index 0000000000..b2f2c4f3aa --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus/unit_to_ucum.go @@ -0,0 +1,90 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package prometheus // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus" + +import "strings" + +var wordToUCUM = map[string]string{ + + // Time + "days": "d", + "hours": "h", + "minutes": "min", + "seconds": "s", + "milliseconds": "ms", + "microseconds": "us", + "nanoseconds": "ns", + + // Bytes + "bytes": "By", + "kibibytes": "KiBy", + "mebibytes": "MiBy", + "gibibytes": "GiBy", + "tibibytes": "TiBy", + "kilobytes": "KBy", + "megabytes": "MBy", + "gigabytes": "GBy", + "terabytes": "TBy", + + // SI + "meters": "m", + "volts": "V", + "amperes": "A", + "joules": "J", + "watts": "W", + "grams": "g", + + // Misc + "celsius": "Cel", + "hertz": "Hz", + "ratio": "1", + "percent": "%", +} + +// The map that translates the "per" unit +// Example: per_second (singular) => /s +var perWordToUCUM = map[string]string{ + "second": "s", + "minute": "m", + "hour": "h", + "day": "d", + "week": "w", + "month": "mo", + "year": "y", +} + +// UnitWordToUCUM converts english unit words to UCUM units: +// https://ucum.org/ucum#section-Alphabetic-Index-By-Symbol +// It also handles rates, such as meters_per_second, by translating the first +// word to UCUM, and the "per" word to UCUM. It joins them with a "/" between. +func UnitWordToUCUM(unit string) string { + unitTokens := strings.SplitN(unit, "_per_", 2) + if len(unitTokens) == 0 { + return "" + } + ucumUnit := wordToUCUMOrDefault(unitTokens[0]) + if len(unitTokens) > 1 && unitTokens[1] != "" { + ucumUnit += "/" + perWordToUCUMOrDefault(unitTokens[1]) + } + return ucumUnit +} + +// wordToUCUMOrDefault retrieves the Prometheus "basic" unit corresponding to +// the specified "basic" unit. Returns the specified unit if not found in +// wordToUCUM. +func wordToUCUMOrDefault(unit string) string { + if promUnit, ok := wordToUCUM[unit]; ok { + return promUnit + } + return unit +} + +// perWordToUCUMOrDefault retrieve the Prometheus "per" unit corresponding to +// the specified "per" unit. Returns the specified unit if not found in perWordToUCUM. +func perWordToUCUMOrDefault(perUnit string) string { + if promPerUnit, ok := perWordToUCUM[perUnit]; ok { + return promPerUnit + } + return perUnit +} diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go index 6080686e76..49ad5672b3 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go @@ -71,8 +71,8 @@ func (a ByLabelName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } // creates a new TimeSeries in the map if not found and returns the time series signature. // tsMap will be unmodified if either labels or sample is nil, but can still be modified if the exemplar is nil. func addSample(tsMap map[string]*prompb.TimeSeries, sample *prompb.Sample, labels []prompb.Label, - datatype string, -) string { + datatype string) string { + if sample == nil || labels == nil || tsMap == nil { return "" } @@ -132,7 +132,14 @@ func addExemplar(tsMap map[string]*prompb.TimeSeries, bucketBounds []bucketBound // the label slice should not contain duplicate label names; this method sorts the slice by label name before creating // the signature. func timeSeriesSignature(datatype string, labels *[]prompb.Label) string { + length := len(datatype) + + for _, lb := range *labels { + length += 2 + len(lb.GetName()) + len(lb.GetValue()) + } + b := strings.Builder{} + b.Grow(length) b.WriteString(datatype) sort.Sort(ByLabelName(*labels)) @@ -151,8 +158,22 @@ func timeSeriesSignature(datatype string, labels *[]prompb.Label) string { // Unpaired string value is ignored. String pairs overwrites OTLP labels if collision happens, and the overwrite is // logged. Resultant label names are sanitized. func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string, extras ...string) []prompb.Label { + serviceName, haveServiceName := resource.Attributes().Get(conventions.AttributeServiceName) + instance, haveInstanceID := resource.Attributes().Get(conventions.AttributeServiceInstanceID) + + // Calculate the maximum possible number of labels we could return so we can preallocate l + maxLabelCount := attributes.Len() + len(externalLabels) + len(extras)/2 + + if haveServiceName { + maxLabelCount++ + } + + if haveInstanceID { + maxLabelCount++ + } + // map ensures no duplicate label name - l := map[string]prompb.Label{} + l := make(map[string]string, maxLabelCount) // Ensure attributes are sorted by key for consistent merging of keys which // collide when sanitized. @@ -164,35 +185,25 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa sort.Stable(ByLabelName(labels)) for _, label := range labels { - finalKey := prometheustranslator.NormalizeLabel(label.Name) + var finalKey = prometheustranslator.NormalizeLabel(label.Name) if existingLabel, alreadyExists := l[finalKey]; alreadyExists { - existingLabel.Value = existingLabel.Value + ";" + label.Value - l[finalKey] = existingLabel + l[finalKey] = existingLabel + ";" + label.Value } else { - l[finalKey] = prompb.Label{ - Name: finalKey, - Value: label.Value, - } + l[finalKey] = label.Value } } // Map service.name + service.namespace to job - if serviceName, ok := resource.Attributes().Get(conventions.AttributeServiceName); ok { + if haveServiceName { val := serviceName.AsString() if serviceNamespace, ok := resource.Attributes().Get(conventions.AttributeServiceNamespace); ok { val = fmt.Sprintf("%s/%s", serviceNamespace.AsString(), val) } - l[model.JobLabel] = prompb.Label{ - Name: model.JobLabel, - Value: val, - } + l[model.JobLabel] = val } // Map service.instance.id to instance - if instance, ok := resource.Attributes().Get(conventions.AttributeServiceInstanceID); ok { - l[model.InstanceLabel] = prompb.Label{ - Name: model.InstanceLabel, - Value: instance.AsString(), - } + if haveInstanceID { + l[model.InstanceLabel] = instance.AsString() } for key, value := range externalLabels { // External labels have already been sanitized @@ -200,10 +211,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa // Skip external labels if they are overridden by metric attributes continue } - l[key] = prompb.Label{ - Name: key, - Value: value, - } + l[key] = value } for i := 0; i < len(extras); i += 2 { @@ -219,15 +227,12 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") { name = prometheustranslator.NormalizeLabel(name) } - l[name] = prompb.Label{ - Name: name, - Value: extras[i+1], - } + l[name] = extras[i+1] } s := make([]prompb.Label, 0, len(l)) - for _, lb := range l { - s = append(s, lb) + for k, v := range l { + s = append(s, prompb.Label{Name: k, Value: v}) } return s @@ -236,6 +241,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa // isValidAggregationTemporality checks whether an OTel metric has a valid // aggregation temporality for conversion to a Prometheus metric. func isValidAggregationTemporality(metric pmetric.Metric) bool { + //exhaustive:enforce switch metric.Type() { case pmetric.MetricTypeGauge, pmetric.MetricTypeSummary: return true @@ -254,7 +260,22 @@ func isValidAggregationTemporality(metric pmetric.Metric) bool { func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon.Resource, metric pmetric.Metric, settings Settings, tsMap map[string]*prompb.TimeSeries) { timestamp := convertTimeStamp(pt.Timestamp()) // sum, count, and buckets of the histogram should append suffix to baseName - baseName := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) + baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels) + + createLabels := func(nameSuffix string, extras ...string) []prompb.Label { + extraLabelCount := len(extras) / 2 + labels := make([]prompb.Label, len(baseLabels), len(baseLabels)+extraLabelCount+1) // +1 for name + copy(labels, baseLabels) + + for extrasIdx := 0; extrasIdx < extraLabelCount; extrasIdx++ { + labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]}) + } + + labels = append(labels, prompb.Label{Name: nameStr, Value: baseName + nameSuffix}) + + return labels + } // If the sum is unset, it indicates the _sum metric point should be // omitted @@ -268,7 +289,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon sum.Value = math.Float64frombits(value.StaleNaN) } - sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr) + sumlabels := createLabels(sumStr) addSample(tsMap, sum, sumlabels, metric.Type().String()) } @@ -282,7 +303,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon count.Value = math.Float64frombits(value.StaleNaN) } - countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr) + countlabels := createLabels(countStr) addSample(tsMap, count, countlabels, metric.Type().String()) // cumulative count for conversion to cumulative histogram @@ -304,7 +325,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon bucket.Value = math.Float64frombits(value.StaleNaN) } boundStr := strconv.FormatFloat(bound, 'f', -1, 64) - labels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, boundStr) + labels := createLabels(bucketStr, leStr, boundStr) sig := addSample(tsMap, bucket, labels, metric.Type().String()) bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: bound}) @@ -318,7 +339,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon } else { infBucket.Value = float64(pt.Count()) } - infLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, pInfStr) + infLabels := createLabels(bucketStr, leStr, pInfStr) sig := addSample(tsMap, infBucket, infLabels, metric.Type().String()) bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: math.Inf(1)}) @@ -327,14 +348,8 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon // add _created time series if needed startTimestamp := pt.StartTimestamp() if settings.ExportCreatedMetric && startTimestamp != 0 { - createdLabels := createAttributes( - resource, - pt.Attributes(), - settings.ExternalLabels, - nameStr, - baseName+createdSuffix, - ) - addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, metric.Type().String()) + labels := createLabels(createdSuffix) + addCreatedTimeSeriesIfNeeded(tsMap, labels, startTimestamp, metric.Type().String()) } } @@ -402,6 +417,7 @@ func getPromExemplars[T exemplarType](pt T) []prompb.Exemplar { func mostRecentTimestampInMetric(metric pmetric.Metric) pcommon.Timestamp { var ts pcommon.Timestamp // handle individual metric based on type + //exhaustive:enforce switch metric.Type() { case pmetric.MetricTypeGauge: dataPoints := metric.Gauge().DataPoints() @@ -441,11 +457,26 @@ func maxTimestamp(a, b pcommon.Timestamp) pcommon.Timestamp { // addSingleSummaryDataPoint converts pt to len(QuantileValues) + 2 samples. func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Resource, metric pmetric.Metric, settings Settings, - tsMap map[string]*prompb.TimeSeries, -) { + tsMap map[string]*prompb.TimeSeries) { timestamp := convertTimeStamp(pt.Timestamp()) // sum and count of the summary should append suffix to baseName - baseName := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) + baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels) + + createLabels := func(name string, extras ...string) []prompb.Label { + extraLabelCount := len(extras) / 2 + labels := make([]prompb.Label, len(baseLabels), len(baseLabels)+extraLabelCount+1) // +1 for name + copy(labels, baseLabels) + + for extrasIdx := 0; extrasIdx < extraLabelCount; extrasIdx++ { + labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]}) + } + + labels = append(labels, prompb.Label{Name: nameStr, Value: name}) + + return labels + } + // treat sum as a sample in an individual TimeSeries sum := &prompb.Sample{ Value: pt.Sum(), @@ -454,7 +485,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res if pt.Flags().NoRecordedValue() { sum.Value = math.Float64frombits(value.StaleNaN) } - sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr) + sumlabels := createLabels(baseName + sumStr) addSample(tsMap, sum, sumlabels, metric.Type().String()) // treat count as a sample in an individual TimeSeries @@ -465,7 +496,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res if pt.Flags().NoRecordedValue() { count.Value = math.Float64frombits(value.StaleNaN) } - countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr) + countlabels := createLabels(baseName + countStr) addSample(tsMap, count, countlabels, metric.Type().String()) // process each percentile/quantile @@ -479,20 +510,14 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res quantile.Value = math.Float64frombits(value.StaleNaN) } percentileStr := strconv.FormatFloat(qt.Quantile(), 'f', -1, 64) - qtlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName, quantileStr, percentileStr) + qtlabels := createLabels(baseName, quantileStr, percentileStr) addSample(tsMap, quantile, qtlabels, metric.Type().String()) } // add _created time series if needed startTimestamp := pt.StartTimestamp() if settings.ExportCreatedMetric && startTimestamp != 0 { - createdLabels := createAttributes( - resource, - pt.Attributes(), - settings.ExternalLabels, - nameStr, - baseName+createdSuffix, - ) + createdLabels := createLabels(baseName + createdSuffix) addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, metric.Type().String()) } } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go index 9a4ec6e11a..3c7494a6bf 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go @@ -60,15 +60,20 @@ func addSingleExponentialHistogramDataPoint( // to Prometheus Native Histogram. func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prompb.Histogram, error) { scale := p.Scale() - if scale < -4 || scale > 8 { + if scale < -4 { return prompb.Histogram{}, fmt.Errorf("cannot convert exponential to native histogram."+ - " Scale must be <= 8 and >= -4, was %d", scale) - // TODO: downscale to 8 if scale > 8 + " Scale must be >= -4, was %d", scale) } - pSpans, pDeltas := convertBucketsLayout(p.Positive()) - nSpans, nDeltas := convertBucketsLayout(p.Negative()) + var scaleDown int32 + if scale > 8 { + scaleDown = scale - 8 + scale = 8 + } + + pSpans, pDeltas := convertBucketsLayout(p.Positive(), scaleDown) + nSpans, nDeltas := convertBucketsLayout(p.Negative(), scaleDown) h := prompb.Histogram{ Schema: scale, @@ -106,17 +111,19 @@ func exponentialToNativeHistogram(p pmetric.ExponentialHistogramDataPoint) (prom // The bucket indexes conversion was adjusted, since OTel exp. histogram bucket // index 0 corresponds to the range (1, base] while Prometheus bucket index 0 // to the range (base 1]. -func convertBucketsLayout(buckets pmetric.ExponentialHistogramDataPointBuckets) ([]prompb.BucketSpan, []int64) { +// +// scaleDown is the factor by which the buckets are scaled down. In other words 2^scaleDown buckets will be merged into one. +func convertBucketsLayout(buckets pmetric.ExponentialHistogramDataPointBuckets, scaleDown int32) ([]prompb.BucketSpan, []int64) { bucketCounts := buckets.BucketCounts() if bucketCounts.Len() == 0 { return nil, nil } var ( - spans []prompb.BucketSpan - deltas []int64 - prevCount int64 - nextBucketIdx int32 + spans []prompb.BucketSpan + deltas []int64 + count int64 + prevCount int64 ) appendDelta := func(count int64) { @@ -125,34 +132,67 @@ func convertBucketsLayout(buckets pmetric.ExponentialHistogramDataPointBuckets) prevCount = count } - for i := 0; i < bucketCounts.Len(); i++ { - count := int64(bucketCounts.At(i)) + // Let the compiler figure out that this is const during this function by + // moving it into a local variable. + numBuckets := bucketCounts.Len() + + // The offset is scaled and adjusted by 1 as described above. + bucketIdx := buckets.Offset()>>scaleDown + 1 + spans = append(spans, prompb.BucketSpan{ + Offset: bucketIdx, + Length: 0, + }) + + for i := 0; i < numBuckets; i++ { + // The offset is scaled and adjusted by 1 as described above. + nextBucketIdx := (int32(i)+buckets.Offset())>>scaleDown + 1 + if bucketIdx == nextBucketIdx { // We have not collected enough buckets to merge yet. + count += int64(bucketCounts.At(i)) + continue + } if count == 0 { + count = int64(bucketCounts.At(i)) continue } - // The offset is adjusted by 1 as described above. - bucketIdx := int32(i) + buckets.Offset() + 1 - delta := bucketIdx - nextBucketIdx - if i == 0 || delta > 2 { - // We have to create a new span, either because we are - // at the very beginning, or because we have found a gap + gap := nextBucketIdx - bucketIdx - 1 + if gap > 2 { + // We have to create a new span, because we have found a gap // of more than two buckets. The constant 2 is copied from the logic in // https://github.com/prometheus/client_golang/blob/27f0506d6ebbb117b6b697d0552ee5be2502c5f2/prometheus/histogram.go#L1296 spans = append(spans, prompb.BucketSpan{ - Offset: delta, + Offset: gap, Length: 0, }) } else { // We have found a small gap (or no gap at all). // Insert empty buckets as needed. - for j := int32(0); j < delta; j++ { + for j := int32(0); j < gap; j++ { appendDelta(0) } } appendDelta(count) - nextBucketIdx = bucketIdx + 1 + count = int64(bucketCounts.At(i)) + bucketIdx = nextBucketIdx + } + // Need to use the last item's index. The offset is scaled and adjusted by 1 as described above. + gap := (int32(numBuckets)+buckets.Offset()-1)>>scaleDown + 1 - bucketIdx + if gap > 2 { + // We have to create a new span, because we have found a gap + // of more than two buckets. The constant 2 is copied from the logic in + // https://github.com/prometheus/client_golang/blob/27f0506d6ebbb117b6b697d0552ee5be2502c5f2/prometheus/histogram.go#L1296 + spans = append(spans, prompb.BucketSpan{ + Offset: gap, + Length: 0, + }) + } else { + // We have found a small gap (or no gap at all). + // Insert empty buckets as needed. + for j := int32(0); j < gap; j++ { + appendDelta(0) + } } + appendDelta(count) return spans, deltas } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go index 34ee762dd4..6a5a656048 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go @@ -22,6 +22,7 @@ type Settings struct { ExternalLabels map[string]string DisableTargetInfo bool ExportCreatedMetric bool + AddMetricSuffixes bool } // FromMetrics converts pmetric.Metrics to prometheus remote write format. @@ -51,6 +52,7 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp } // handle individual metric based on type + //exhaustive:enforce switch metric.Type() { case pmetric.MetricTypeGauge: dataPoints := metric.Gauge().DataPoints() @@ -81,7 +83,7 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp if dataPoints.Len() == 0 { errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) } - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) for x := 0; x < dataPoints.Len(); x++ { errs = multierr.Append( errs, diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go index 3a5d201ddd..c8e59694b8 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/number_data_points.go @@ -27,7 +27,7 @@ func addSingleGaugeNumberDataPoint( settings Settings, series map[string]*prompb.TimeSeries, ) { - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) labels := createAttributes( resource, pt.Attributes(), @@ -60,7 +60,7 @@ func addSingleSumNumberDataPoint( settings Settings, series map[string]*prompb.TimeSeries, ) { - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) labels := createAttributes( resource, pt.Attributes(), diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/read_handler.go b/vendor/github.com/prometheus/prometheus/storage/remote/read_handler.go index e2702c9f77..3a99e3360c 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/read_handler.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/read_handler.go @@ -15,6 +15,7 @@ package remote import ( "context" + "errors" "net/http" "strings" "sync" @@ -169,7 +170,8 @@ func (h *readHandler) remoteReadSamples( } return nil }(); err != nil { - if httpErr, ok := err.(HTTPError); ok { + var httpErr HTTPError + if errors.As(err, &httpErr) { http.Error(w, httpErr.Error(), httpErr.Status()) return } @@ -241,7 +243,8 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re } return nil }(); err != nil { - if httpErr, ok := err.(HTTPError); ok { + var httpErr HTTPError + if errors.As(err, &httpErr) { http.Error(w, httpErr.Error(), httpErr.Status()) return } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go index b6533f9275..758ba3cc91 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go @@ -77,10 +77,7 @@ func NewStorage(l log.Logger, reg prometheus.Registerer, stCallback startTimeCal } func (s *Storage) Notify() { - for _, q := range s.rws.queues { - // These should all be non blocking - q.watcher.Notify() - } + s.rws.Notify() } // ApplyConfig updates the state as the new config requires. diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/write.go b/vendor/github.com/prometheus/prometheus/storage/remote/write.go index 4b0a249014..237f8caa91 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/write.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/write.go @@ -121,6 +121,16 @@ func (rws *WriteStorage) run() { } } +func (rws *WriteStorage) Notify() { + rws.mtx.Lock() + defer rws.mtx.Unlock() + + for _, q := range rws.queues { + // These should all be non blocking + q.watcher.Notify() + } +} + // ApplyConfig updates the state as the new config requires. // Only stop & create queues which have changes. func (rws *WriteStorage) ApplyConfig(conf *config.Config) error { diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go b/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go index 6c0cd8a29b..9891c6aae7 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/write_handler.go @@ -66,9 +66,9 @@ func (h *writeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } err = h.write(r.Context(), req) - switch err { - case nil: - case storage.ErrOutOfOrderSample, storage.ErrOutOfBounds, storage.ErrDuplicateSampleForTimestamp: + switch { + case err == nil: + case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrOutOfBounds), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): // Indicated an out of order sample is a bad request to prevent retries. http.Error(w, err.Error(), http.StatusBadRequest) return @@ -207,7 +207,9 @@ func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - prwMetricsMap, errs := otlptranslator.FromMetrics(req.Metrics(), otlptranslator.Settings{}) + prwMetricsMap, errs := otlptranslator.FromMetrics(req.Metrics(), otlptranslator.Settings{ + AddMetricSuffixes: true, + }) if errs != nil { level.Warn(h.logger).Log("msg", "Error translating OTLP metrics to Prometheus write request", "err", errs) } @@ -222,9 +224,9 @@ func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Timeseries: prwMetrics, }) - switch err { - case nil: - case storage.ErrOutOfOrderSample, storage.ErrOutOfBounds, storage.ErrDuplicateSampleForTimestamp: + switch { + case err == nil: + case errors.Is(err, storage.ErrOutOfOrderSample), errors.Is(err, storage.ErrOutOfBounds), errors.Is(err, storage.ErrDuplicateSampleForTimestamp): // Indicated an out of order sample is a bad request to prevent retries. http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/vendor/github.com/prometheus/prometheus/template/template.go b/vendor/github.com/prometheus/prometheus/template/template.go index 01f6ec9a8a..5d72a7e83e 100644 --- a/vendor/github.com/prometheus/prometheus/template/template.go +++ b/vendor/github.com/prometheus/prometheus/template/template.go @@ -181,7 +181,7 @@ func NewTemplateExpander( return html_template.HTML(text) }, "match": regexp.MatchString, - "title": strings.Title, // nolint:staticcheck + "title": strings.Title, //nolint:staticcheck "toUpper": strings.ToUpper, "toLower": strings.ToLower, "graphLink": strutil.GraphLinkForExpression, diff --git a/vendor/github.com/prometheus/prometheus/tsdb/block.go b/vendor/github.com/prometheus/prometheus/tsdb/block.go index 13a3899702..a586536b15 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/block.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/block.go @@ -17,6 +17,7 @@ package tsdb import ( "context" "encoding/json" + "fmt" "io" "os" "path/filepath" @@ -116,8 +117,19 @@ type ChunkWriter interface { // ChunkReader provides reading access of serialized time series data. type ChunkReader interface { - // Chunk returns the series data chunk with the given reference. - Chunk(meta chunks.Meta) (chunkenc.Chunk, error) + // ChunkOrIterable returns the series data for the given chunks.Meta. + // Either a single chunk will be returned, or an iterable. + // A single chunk should be returned if chunks.Meta maps to a chunk that + // already exists and doesn't need modifications. + // An iterable should be returned if chunks.Meta maps to a subset of the + // samples in a stored chunk, or multiple chunks. (E.g. OOOHeadChunkReader + // could return an iterable where multiple histogram samples have counter + // resets. There can only be one counter reset per histogram chunk so + // multiple chunks would be created from the iterable in this case.) + // Only one of chunk or iterable should be returned. In some cases you may + // always expect a chunk to be returned. You can check that iterable is nil + // in those cases. + ChunkOrIterable(meta chunks.Meta) (chunkenc.Chunk, chunkenc.Iterable, error) // Close releases all underlying resources of the reader. Close() error @@ -238,7 +250,7 @@ func readMetaFile(dir string) (*BlockMeta, int64, error) { return nil, 0, err } if m.Version != metaVersion1 { - return nil, 0, errors.Errorf("unexpected meta file version %d", m.Version) + return nil, 0, fmt.Errorf("unexpected meta file version %d", m.Version) } return &m, int64(len(b)), nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go index e7ff5b165e..0126f1fbdb 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/chunk.go @@ -14,11 +14,10 @@ package chunkenc import ( + "fmt" "math" "sync" - "github.com/pkg/errors" - "github.com/prometheus/prometheus/model/histogram" ) @@ -68,6 +67,8 @@ const ( // Chunk holds a sequence of sample pairs that can be iterated over and appended to. type Chunk interface { + Iterable + // Bytes returns the underlying byte slice of the chunk. Bytes() []byte @@ -77,11 +78,6 @@ type Chunk interface { // Appender returns an appender to append samples to the chunk. Appender() (Appender, error) - // The iterator passed as argument is for re-use. - // Depending on implementation, the iterator can - // be re-used or a new iterator can be allocated. - Iterator(Iterator) Iterator - // NumSamples returns the number of samples in the chunk. NumSamples() int @@ -93,6 +89,13 @@ type Chunk interface { Compact() } +type Iterable interface { + // The iterator passed as argument is for re-use. + // Depending on implementation, the iterator can + // be re-used or a new iterator can be allocated. + Iterator(Iterator) Iterator +} + // Appender adds sample pairs to a chunk. type Appender interface { Append(int64, float64) @@ -185,6 +188,19 @@ func (v ValueType) ChunkEncoding() Encoding { } } +func (v ValueType) NewChunk() (Chunk, error) { + switch v { + case ValFloat: + return NewXORChunk(), nil + case ValHistogram: + return NewHistogramChunk(), nil + case ValFloatHistogram: + return NewFloatHistogramChunk(), nil + default: + return nil, fmt.Errorf("value type %v unsupported", v) + } +} + // MockSeriesIterator returns an iterator for a mock series with custom timeStamps and values. func MockSeriesIterator(timestamps []int64, values []float64) Iterator { return &mockSeriesIterator{ @@ -293,7 +309,7 @@ func (p *pool) Get(e Encoding, b []byte) (Chunk, error) { c.b.count = 0 return c, nil } - return nil, errors.Errorf("invalid chunk encoding %q", e) + return nil, fmt.Errorf("invalid chunk encoding %q", e) } func (p *pool) Put(c Chunk) error { @@ -332,7 +348,7 @@ func (p *pool) Put(c Chunk) error { sh.b.count = 0 p.floatHistogram.Put(c) default: - return errors.Errorf("invalid chunk encoding %q", c.Encoding()) + return fmt.Errorf("invalid chunk encoding %q", c.Encoding()) } return nil } @@ -349,7 +365,7 @@ func FromData(e Encoding, d []byte) (Chunk, error) { case EncFloatHistogram: return &FloatHistogramChunk{b: bstream{count: 0, stream: d}}, nil } - return nil, errors.Errorf("invalid chunk encoding %q", e) + return nil, fmt.Errorf("invalid chunk encoding %q", e) } // NewEmptyChunk returns an empty chunk for the given encoding. @@ -362,5 +378,5 @@ func NewEmptyChunk(e Encoding) (Chunk, error) { case EncFloatHistogram: return NewFloatHistogramChunk(), nil } - return nil, errors.Errorf("invalid chunk encoding %q", e) + return nil, fmt.Errorf("invalid chunk encoding %q", e) } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/float_histogram.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/float_histogram.go index 505d112455..3d76cdf65f 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/float_histogram.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/float_histogram.go @@ -103,7 +103,7 @@ func (c *FloatHistogramChunk) Appender() (Appender, error) { // To get an appender, we must know the state it would have if we had // appended all existing data from scratch. We iterate through the end // and populate via the iterator's state. - for it.Next() == ValFloatHistogram { // nolint:revive + for it.Next() == ValFloatHistogram { } if err := it.Err(); err != nil { return nil, err @@ -605,10 +605,10 @@ func (a *FloatHistogramAppender) AppendFloatHistogram(prev *FloatHistogramAppend pForwardInserts, nForwardInserts, okToAppend, counterReset := a.appendable(h) if !okToAppend || counterReset { if appendOnly { - if !okToAppend { - return nil, false, a, fmt.Errorf("float histogram schema change") + if counterReset { + return nil, false, a, fmt.Errorf("float histogram counter reset") } - return nil, false, a, fmt.Errorf("float histogram counter reset") + return nil, false, a, fmt.Errorf("float histogram schema change") } newChunk := NewFloatHistogramChunk() app, err := newChunk.Appender() diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram.go index 847d893761..ac84e7a1ef 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram.go @@ -114,7 +114,7 @@ func (c *HistogramChunk) Appender() (Appender, error) { // To get an appender, we must know the state it would have if we had // appended all existing data from scratch. We iterate through the end // and populate via the iterator's state. - for it.Next() == ValHistogram { // nolint:revive + for it.Next() == ValHistogram { } if err := it.Err(); err != nil { return nil, err @@ -640,10 +640,10 @@ func (a *HistogramAppender) AppendHistogram(prev *HistogramAppender, t int64, h pForwardInserts, nForwardInserts, okToAppend, counterReset := a.appendable(h) if !okToAppend || counterReset { if appendOnly { - if !okToAppend { - return nil, false, a, fmt.Errorf("histogram schema change") + if counterReset { + return nil, false, a, fmt.Errorf("histogram counter reset") } - return nil, false, a, fmt.Errorf("histogram counter reset") + return nil, false, a, fmt.Errorf("histogram schema change") } newChunk := NewHistogramChunk() app, err := newChunk.Appender() diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram_meta.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram_meta.go index cda1080a8b..70f129b953 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram_meta.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/histogram_meta.go @@ -284,7 +284,7 @@ loop: // cover an entirely different set of buckets. The function returns the // “forward” inserts to expand 'a' to also cover all the buckets exclusively // covered by 'b', and it returns the “backward” inserts to expand 'b' to also -// cover all the buckets exclusively covered by 'a' +// cover all the buckets exclusively covered by 'a'. func expandSpansBothWays(a, b []histogram.Span) (forward, backward []Insert, mergedSpans []histogram.Span) { ai := newBucketIterator(a) bi := newBucketIterator(b) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/varbit.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/varbit.go index 449f9fbac2..b43574dcb6 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/varbit.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/varbit.go @@ -14,9 +14,8 @@ package chunkenc import ( + "fmt" "math/bits" - - "github.com/pkg/errors" ) // putVarbitInt writes an int64 using varbit encoding with a bit bucketing @@ -109,7 +108,7 @@ func readVarbitInt(b *bstreamReader) (int64, error) { val = int64(bits) default: - return 0, errors.Errorf("invalid bit pattern %b", d) + return 0, fmt.Errorf("invalid bit pattern %b", d) } if sz != 0 { @@ -215,7 +214,7 @@ func readVarbitUint(b *bstreamReader) (uint64, error) { return 0, err } default: - return 0, errors.Errorf("invalid bit pattern %b", d) + return 0, fmt.Errorf("invalid bit pattern %b", d) } if sz != 0 { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go index 089a51a644..d54e5dbab1 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunkenc/xor.go @@ -99,7 +99,7 @@ func (c *XORChunk) Appender() (Appender, error) { // To get an appender we must know the state it would have if we had // appended all existing data from scratch. // We iterate through the end and populate via the iterator's state. - for it.Next() != ValNone { // nolint:revive + for it.Next() != ValNone { } if err := it.Err(); err != nil { return nil, err diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go index 05fd24a062..f22285a0ca 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/chunks.go @@ -24,8 +24,6 @@ import ( "path/filepath" "strconv" - "github.com/pkg/errors" - "github.com/prometheus/prometheus/tsdb/chunkenc" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" "github.com/prometheus/prometheus/tsdb/fileutil" @@ -93,13 +91,14 @@ func (p HeadChunkRef) Unpack() (HeadSeriesRef, HeadChunkID) { // // Example: // assume a memSeries.firstChunkID=7 and memSeries.mmappedChunks=[p5,p6,p7,p8,p9]. -// | HeadChunkID value | refers to ... | -// |-------------------|----------------------------------------------------------------------------------------| -// | 0-6 | chunks that have been compacted to blocks, these won't return data for queries in Head | -// | 7-11 | memSeries.mmappedChunks[i] where i is 0 to 4. | -// | 12 | *memChunk{next: nil} -// | 13 | *memChunk{next: ^} -// | 14 | memSeries.headChunks -> *memChunk{next: ^} +// +// | HeadChunkID value | refers to ... | +// |-------------------|----------------------------------------------------------------------------------------| +// | 0-6 | chunks that have been compacted to blocks, these won't return data for queries in Head | +// | 7-11 | memSeries.mmappedChunks[i] where i is 0 to 4. | +// | 12 | *memChunk{next: nil} +// | 13 | *memChunk{next: ^} +// | 14 | memSeries.headChunks -> *memChunk{next: ^} type HeadChunkID uint64 // BlockChunkRef refers to a chunk within a persisted block. @@ -118,11 +117,16 @@ func (b BlockChunkRef) Unpack() (int, int) { return sgmIndex, chkStart } -// Meta holds information about a chunk of data. +// Meta holds information about one or more chunks. +// For examples of when chunks.Meta could refer to multiple chunks, see +// ChunkReader.ChunkOrIterable(). type Meta struct { // Ref and Chunk hold either a reference that can be used to retrieve // chunk data or the data itself. - // If Chunk is nil, call ChunkReader.Chunk(Meta.Ref) to get the chunk and assign it to the Chunk field + // If Chunk is nil, call ChunkReader.ChunkOrIterable(Meta.Ref) to get the + // chunk and assign it to the Chunk field. If an iterable is returned from + // that method, then it may not be possible to set Chunk as the iterable + // might form several chunks. Ref ChunkRef Chunk chunkenc.Chunk @@ -198,7 +202,7 @@ func ChunkFromSamplesGeneric(s Samples) (Meta, error) { }, nil } -// PopulatedChunk creates a chunk populated with samples every second starting at minTime +// PopulatedChunk creates a chunk populated with samples every second starting at minTime. func PopulatedChunk(numSamples int, minTime int64) (Meta, error) { samples := make([]Sample, numSamples) for i := 0; i < numSamples; i++ { @@ -284,7 +288,7 @@ func checkCRC32(data, sum []byte) error { // This combination of shifts is the inverse of digest.Sum() in go/src/hash/crc32. want := uint32(sum[0])<<24 + uint32(sum[1])<<16 + uint32(sum[2])<<8 + uint32(sum[3]) if got != want { - return errors.Errorf("checksum mismatch expected:%x, actual:%x", want, got) + return fmt.Errorf("checksum mismatch expected:%x, actual:%x", want, got) } return nil } @@ -397,12 +401,12 @@ func (w *Writer) cut() error { func cutSegmentFile(dirFile *os.File, magicNumber uint32, chunksFormat byte, allocSize int64) (headerSize int, newFile *os.File, seq int, returnErr error) { p, seq, err := nextSequenceFile(dirFile.Name()) if err != nil { - return 0, nil, 0, errors.Wrap(err, "next sequence file") + return 0, nil, 0, fmt.Errorf("next sequence file: %w", err) } ptmp := p + ".tmp" f, err := os.OpenFile(ptmp, os.O_WRONLY|os.O_CREATE, 0o666) if err != nil { - return 0, nil, 0, errors.Wrap(err, "open temp file") + return 0, nil, 0, fmt.Errorf("open temp file: %w", err) } defer func() { if returnErr != nil { @@ -417,11 +421,11 @@ func cutSegmentFile(dirFile *os.File, magicNumber uint32, chunksFormat byte, all }() if allocSize > 0 { if err = fileutil.Preallocate(f, allocSize, true); err != nil { - return 0, nil, 0, errors.Wrap(err, "preallocate") + return 0, nil, 0, fmt.Errorf("preallocate: %w", err) } } if err = dirFile.Sync(); err != nil { - return 0, nil, 0, errors.Wrap(err, "sync directory") + return 0, nil, 0, fmt.Errorf("sync directory: %w", err) } // Write header metadata for new file. @@ -431,24 +435,24 @@ func cutSegmentFile(dirFile *os.File, magicNumber uint32, chunksFormat byte, all n, err := f.Write(metab) if err != nil { - return 0, nil, 0, errors.Wrap(err, "write header") + return 0, nil, 0, fmt.Errorf("write header: %w", err) } if err := f.Close(); err != nil { - return 0, nil, 0, errors.Wrap(err, "close temp file") + return 0, nil, 0, fmt.Errorf("close temp file: %w", err) } f = nil if err := fileutil.Rename(ptmp, p); err != nil { - return 0, nil, 0, errors.Wrap(err, "replace file") + return 0, nil, 0, fmt.Errorf("replace file: %w", err) } f, err = os.OpenFile(p, os.O_WRONLY, 0o666) if err != nil { - return 0, nil, 0, errors.Wrap(err, "open final file") + return 0, nil, 0, fmt.Errorf("open final file: %w", err) } // Skip header for further writes. if _, err := f.Seek(int64(n), 0); err != nil { - return 0, nil, 0, errors.Wrap(err, "seek in final file") + return 0, nil, 0, fmt.Errorf("seek in final file: %w", err) } return n, f, seq, nil } @@ -605,16 +609,16 @@ func newReader(bs []ByteSlice, cs []io.Closer, pool chunkenc.Pool) (*Reader, err cr := Reader{pool: pool, bs: bs, cs: cs} for i, b := range cr.bs { if b.Len() < SegmentHeaderSize { - return nil, errors.Wrapf(errInvalidSize, "invalid segment header in segment %d", i) + return nil, fmt.Errorf("invalid segment header in segment %d: %w", i, errInvalidSize) } // Verify magic number. if m := binary.BigEndian.Uint32(b.Range(0, MagicChunksSize)); m != MagicChunks { - return nil, errors.Errorf("invalid magic number %x", m) + return nil, fmt.Errorf("invalid magic number %x", m) } // Verify chunk format version. if v := int(b.Range(MagicChunksSize, MagicChunksSize+ChunksFormatVersionSize)[0]); v != chunksFormatV1 { - return nil, errors.Errorf("invalid chunk format version %d", v) + return nil, fmt.Errorf("invalid chunk format version %d", v) } cr.size += int64(b.Len()) } @@ -640,7 +644,7 @@ func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error) { f, err := fileutil.OpenMmapFile(fn) if err != nil { return nil, tsdb_errors.NewMulti( - errors.Wrap(err, "mmap files"), + fmt.Errorf("mmap files: %w", err), tsdb_errors.CloseAll(cs), ).Err() } @@ -668,24 +672,24 @@ func (s *Reader) Size() int64 { } // Chunk returns a chunk from a given reference. -func (s *Reader) Chunk(meta Meta) (chunkenc.Chunk, error) { +func (s *Reader) ChunkOrIterable(meta Meta) (chunkenc.Chunk, chunkenc.Iterable, error) { sgmIndex, chkStart := BlockChunkRef(meta.Ref).Unpack() if sgmIndex >= len(s.bs) { - return nil, errors.Errorf("segment index %d out of range", sgmIndex) + return nil, nil, fmt.Errorf("segment index %d out of range", sgmIndex) } sgmBytes := s.bs[sgmIndex] if chkStart+MaxChunkLengthFieldSize > sgmBytes.Len() { - return nil, errors.Errorf("segment doesn't include enough bytes to read the chunk size data field - required:%v, available:%v", chkStart+MaxChunkLengthFieldSize, sgmBytes.Len()) + return nil, nil, fmt.Errorf("segment doesn't include enough bytes to read the chunk size data field - required:%v, available:%v", chkStart+MaxChunkLengthFieldSize, sgmBytes.Len()) } // With the minimum chunk length this should never cause us reading // over the end of the slice. c := sgmBytes.Range(chkStart, chkStart+MaxChunkLengthFieldSize) chkDataLen, n := binary.Uvarint(c) if n <= 0 { - return nil, errors.Errorf("reading chunk length failed with %d", n) + return nil, nil, fmt.Errorf("reading chunk length failed with %d", n) } chkEncStart := chkStart + n @@ -694,17 +698,18 @@ func (s *Reader) Chunk(meta Meta) (chunkenc.Chunk, error) { chkDataEnd := chkEnd - crc32.Size if chkEnd > sgmBytes.Len() { - return nil, errors.Errorf("segment doesn't include enough bytes to read the chunk - required:%v, available:%v", chkEnd, sgmBytes.Len()) + return nil, nil, fmt.Errorf("segment doesn't include enough bytes to read the chunk - required:%v, available:%v", chkEnd, sgmBytes.Len()) } sum := sgmBytes.Range(chkDataEnd, chkEnd) if err := checkCRC32(sgmBytes.Range(chkEncStart, chkDataEnd), sum); err != nil { - return nil, err + return nil, nil, err } chkData := sgmBytes.Range(chkDataStart, chkDataEnd) chkEnc := sgmBytes.Range(chkEncStart, chkEncStart+ChunkEncodingSize)[0] - return s.pool.Get(chunkenc.Encoding(chkEnc), chkData) + chk, err := s.pool.Get(chunkenc.Encoding(chkEnc), chkData) + return chk, nil, err } func nextSequenceFile(dir string) (string, int, error) { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go index d73eb36f87..b495b61828 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/chunks/head_chunks.go @@ -17,6 +17,8 @@ import ( "bufio" "bytes" "encoding/binary" + "errors" + "fmt" "hash" "io" "os" @@ -25,7 +27,6 @@ import ( "sync" "github.com/dennwc/varint" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "go.uber.org/atomic" "golang.org/x/exp/slices" @@ -107,7 +108,7 @@ type CorruptionErr struct { } func (e *CorruptionErr) Error() string { - return errors.Wrapf(e.Err, "corruption in head chunk file %s", segmentFile(e.Dir, e.FileIndex)).Error() + return fmt.Errorf("corruption in head chunk file %s: %w", segmentFile(e.Dir, e.FileIndex), e.Err).Error() } // chunkPos keeps track of the position in the head chunk files. @@ -240,10 +241,10 @@ type mmappedChunkFile struct { func NewChunkDiskMapper(reg prometheus.Registerer, dir string, pool chunkenc.Pool, writeBufferSize, writeQueueSize int) (*ChunkDiskMapper, error) { // Validate write buffer size. if writeBufferSize < MinWriteBufferSize || writeBufferSize > MaxWriteBufferSize { - return nil, errors.Errorf("ChunkDiskMapper write buffer size should be between %d and %d (actual: %d)", MinWriteBufferSize, MaxWriteBufferSize, writeBufferSize) + return nil, fmt.Errorf("ChunkDiskMapper write buffer size should be between %d and %d (actual: %d)", MinWriteBufferSize, MaxWriteBufferSize, writeBufferSize) } if writeBufferSize%1024 != 0 { - return nil, errors.Errorf("ChunkDiskMapper write buffer size should be a multiple of 1024 (actual: %d)", writeBufferSize) + return nil, fmt.Errorf("ChunkDiskMapper write buffer size should be a multiple of 1024 (actual: %d)", writeBufferSize) } if err := os.MkdirAll(dir, 0o777); err != nil { @@ -320,7 +321,7 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) { for seq, fn := range files { f, err := fileutil.OpenMmapFile(fn) if err != nil { - return errors.Wrapf(err, "mmap files, file: %s", fn) + return fmt.Errorf("mmap files, file: %s: %w", fn, err) } cdm.closers[seq] = f cdm.mmappedChunkFiles[seq] = &mmappedChunkFile{byteSlice: realByteSlice(f.Bytes())} @@ -335,23 +336,23 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) { lastSeq := chkFileIndices[0] for _, seq := range chkFileIndices[1:] { if seq != lastSeq+1 { - return errors.Errorf("found unsequential head chunk files %s (index: %d) and %s (index: %d)", files[lastSeq], lastSeq, files[seq], seq) + return fmt.Errorf("found unsequential head chunk files %s (index: %d) and %s (index: %d)", files[lastSeq], lastSeq, files[seq], seq) } lastSeq = seq } for i, b := range cdm.mmappedChunkFiles { if b.byteSlice.Len() < HeadChunkFileHeaderSize { - return errors.Wrapf(errInvalidSize, "%s: invalid head chunk file header", files[i]) + return fmt.Errorf("%s: invalid head chunk file header: %w", files[i], errInvalidSize) } // Verify magic number. if m := binary.BigEndian.Uint32(b.byteSlice.Range(0, MagicChunksSize)); m != MagicHeadChunks { - return errors.Errorf("%s: invalid magic number %x", files[i], m) + return fmt.Errorf("%s: invalid magic number %x", files[i], m) } // Verify chunk format version. if v := int(b.byteSlice.Range(MagicChunksSize, MagicChunksSize+ChunksFormatVersionSize)[0]); v != chunksFormatV1 { - return errors.Errorf("%s: invalid chunk format version %d", files[i], v) + return fmt.Errorf("%s: invalid chunk format version %d", files[i], v) } } @@ -394,16 +395,16 @@ func repairLastChunkFile(files map[int]string) (_ map[int]string, returnErr erro f, err := os.Open(files[lastFile]) if err != nil { - return files, errors.Wrap(err, "open file during last head chunk file repair") + return files, fmt.Errorf("open file during last head chunk file repair: %w", err) } buf := make([]byte, MagicChunksSize) size, err := f.Read(buf) if err != nil && err != io.EOF { - return files, errors.Wrap(err, "failed to read magic number during last head chunk file repair") + return files, fmt.Errorf("failed to read magic number during last head chunk file repair: %w", err) } if err := f.Close(); err != nil { - return files, errors.Wrap(err, "close file during last head chunk file repair") + return files, fmt.Errorf("close file during last head chunk file repair: %w", err) } // We either don't have enough bytes for the magic number or the magic number is 0. @@ -413,7 +414,7 @@ func repairLastChunkFile(files map[int]string) (_ map[int]string, returnErr erro if size < MagicChunksSize || binary.BigEndian.Uint32(buf) == 0 { // Corrupt file, hence remove it. if err := os.RemoveAll(files[lastFile]); err != nil { - return files, errors.Wrap(err, "delete corrupted, empty head chunk file during last file repair") + return files, fmt.Errorf("delete corrupted, empty head chunk file during last file repair: %w", err) } delete(files, lastFile) } @@ -559,7 +560,7 @@ func (cdm *ChunkDiskMapper) cutAndExpectRef(chkRef ChunkDiskMapperRef) (err erro } if expSeq, expOffset := chkRef.Unpack(); seq != expSeq || offset != expOffset { - return errors.Errorf("expected newly cut file to have sequence:offset %d:%d, got %d:%d", expSeq, expOffset, seq, offset) + return fmt.Errorf("expected newly cut file to have sequence:offset %d:%d, got %d:%d", expSeq, expOffset, seq, offset) } return nil @@ -701,13 +702,13 @@ func (cdm *ChunkDiskMapper) Chunk(ref ChunkDiskMapperRef) (chunkenc.Chunk, error return nil, &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: -1, - Err: errors.Errorf("head chunk file index %d more than current open file", sgmIndex), + Err: fmt.Errorf("head chunk file index %d more than current open file", sgmIndex), } } return nil, &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: sgmIndex, - Err: errors.Errorf("head chunk file index %d does not exist on disk", sgmIndex), + Err: fmt.Errorf("head chunk file index %d does not exist on disk", sgmIndex), } } @@ -715,7 +716,7 @@ func (cdm *ChunkDiskMapper) Chunk(ref ChunkDiskMapperRef) (chunkenc.Chunk, error return nil, &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: sgmIndex, - Err: errors.Errorf("head chunk file doesn't include enough bytes to read the chunk size data field - required:%v, available:%v", chkStart+MaxChunkLengthFieldSize, mmapFile.byteSlice.Len()), + Err: fmt.Errorf("head chunk file doesn't include enough bytes to read the chunk size data field - required:%v, available:%v", chkStart+MaxChunkLengthFieldSize, mmapFile.byteSlice.Len()), } } @@ -734,7 +735,7 @@ func (cdm *ChunkDiskMapper) Chunk(ref ChunkDiskMapperRef) (chunkenc.Chunk, error return nil, &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: sgmIndex, - Err: errors.Errorf("reading chunk length failed with %d", n), + Err: fmt.Errorf("reading chunk length failed with %d", n), } } @@ -744,7 +745,7 @@ func (cdm *ChunkDiskMapper) Chunk(ref ChunkDiskMapperRef) (chunkenc.Chunk, error return nil, &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: sgmIndex, - Err: errors.Errorf("head chunk file doesn't include enough bytes to read the chunk - required:%v, available:%v", chkDataEnd, mmapFile.byteSlice.Len()), + Err: fmt.Errorf("head chunk file doesn't include enough bytes to read the chunk - required:%v, available:%v", chkDataEnd, mmapFile.byteSlice.Len()), } } @@ -761,7 +762,7 @@ func (cdm *ChunkDiskMapper) Chunk(ref ChunkDiskMapperRef) (chunkenc.Chunk, error return nil, &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: sgmIndex, - Err: errors.Errorf("checksum mismatch expected:%x, actual:%x", sum, act), + Err: fmt.Errorf("checksum mismatch expected:%x, actual:%x", sum, act), } } @@ -829,7 +830,7 @@ func (cdm *ChunkDiskMapper) IterateAllChunks(f func(seriesRef HeadSeriesRef, chu return &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: segID, - Err: errors.Errorf("head chunk file has some unread data, but doesn't include enough bytes to read the chunk header"+ + Err: fmt.Errorf("head chunk file has some unread data, but doesn't include enough bytes to read the chunk header"+ " - required:%v, available:%v, file:%d", idx+MaxHeadChunkMetaSize, fileEnd, segID), } } @@ -866,7 +867,7 @@ func (cdm *ChunkDiskMapper) IterateAllChunks(f func(seriesRef HeadSeriesRef, chu return &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: segID, - Err: errors.Errorf("head chunk file doesn't include enough bytes to read the chunk header - required:%v, available:%v, file:%d", idx+CRCSize, fileEnd, segID), + Err: fmt.Errorf("head chunk file doesn't include enough bytes to read the chunk header - required:%v, available:%v, file:%d", idx+CRCSize, fileEnd, segID), } } @@ -879,7 +880,7 @@ func (cdm *ChunkDiskMapper) IterateAllChunks(f func(seriesRef HeadSeriesRef, chu return &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: segID, - Err: errors.Errorf("checksum mismatch expected:%x, actual:%x", sum, act), + Err: fmt.Errorf("checksum mismatch expected:%x, actual:%x", sum, act), } } idx += CRCSize @@ -905,7 +906,7 @@ func (cdm *ChunkDiskMapper) IterateAllChunks(f func(seriesRef HeadSeriesRef, chu return &CorruptionErr{ Dir: cdm.dir.Name(), FileIndex: segID, - Err: errors.Errorf("head chunk file doesn't include enough bytes to read the last chunk data - required:%v, available:%v, file:%d", idx, fileEnd, segID), + Err: fmt.Errorf("head chunk file doesn't include enough bytes to read the last chunk data - required:%v, available:%v, file:%d", idx, fileEnd, segID), } } } @@ -998,10 +999,9 @@ func (cdm *ChunkDiskMapper) deleteFiles(removedFiles []int) ([]int, error) { // DeleteCorrupted deletes all the head chunk files after the one which had the corruption // (including the corrupt file). func (cdm *ChunkDiskMapper) DeleteCorrupted(originalErr error) error { - err := errors.Cause(originalErr) // So that we can pick up errors even if wrapped. - cerr, ok := err.(*CorruptionErr) - if !ok { - return errors.Wrap(originalErr, "cannot handle error") + var cerr *CorruptionErr + if !errors.As(originalErr, &cerr) { + return fmt.Errorf("cannot handle error: %w", originalErr) } // Delete all the head chunk files following the corrupt head chunk file. diff --git a/vendor/github.com/prometheus/prometheus/tsdb/compact.go b/vendor/github.com/prometheus/prometheus/tsdb/compact.go index ca3a950960..32c88d2cc0 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/compact.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/compact.go @@ -151,7 +151,7 @@ func NewLeveledCompactor(ctx context.Context, r prometheus.Registerer, l log.Log func NewLeveledCompactorWithChunkSize(ctx context.Context, r prometheus.Registerer, l log.Logger, ranges []int64, pool chunkenc.Pool, maxBlockChunkSegmentSize int64, mergeFunc storage.VerticalChunkSeriesMergeFunc) (*LeveledCompactor, error) { if len(ranges) == 0 { - return nil, errors.Errorf("at least one range must be provided") + return nil, fmt.Errorf("at least one range must be provided") } if pool == nil { pool = chunkenc.NewPool() @@ -201,7 +201,14 @@ func (c *LeveledCompactor) Plan(dir string) ([]string, error) { func (c *LeveledCompactor) plan(dms []dirMeta) ([]string, error) { slices.SortFunc(dms, func(a, b dirMeta) int { - return int(a.meta.MinTime - b.meta.MinTime) + switch { + case a.meta.MinTime < b.meta.MinTime: + return -1 + case a.meta.MinTime > b.meta.MinTime: + return 1 + default: + return 0 + } }) res := c.selectOverlappingDirs(dms) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/db.go b/vendor/github.com/prometheus/prometheus/tsdb/db.go index ce8ef1f9aa..2e3801a9e0 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/db.go @@ -203,10 +203,14 @@ type DB struct { compactor Compactor blocksToDelete BlocksToDeleteFunc - // Mutex for that must be held when modifying the general block layout. + // Mutex for that must be held when modifying the general block layout or lastGarbageCollectedMmapRef. mtx sync.RWMutex blocks []*Block + // The last OOO chunk that was compacted and written to disk. New queriers must not read chunks less + // than or equal to this reference, as these chunks could be garbage collected at any time. + lastGarbageCollectedMmapRef chunks.ChunkDiskMapperRef + head *Head compactc chan struct{} @@ -248,6 +252,7 @@ type dbMetrics struct { tombCleanTimer prometheus.Histogram blocksBytes prometheus.Gauge maxBytes prometheus.Gauge + retentionDuration prometheus.Gauge } func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { @@ -321,6 +326,10 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { Name: "prometheus_tsdb_retention_limit_bytes", Help: "Max number of bytes to be retained in the tsdb blocks, configured 0 means disabled", }) + m.retentionDuration = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "prometheus_tsdb_retention_limit_seconds", + Help: "How long to retain samples in storage.", + }) m.sizeRetentionCount = prometheus.NewCounter(prometheus.CounterOpts{ Name: "prometheus_tsdb_size_retentions_total", Help: "The number of times that blocks were deleted because the maximum number of bytes was exceeded.", @@ -341,6 +350,7 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { m.tombCleanTimer, m.blocksBytes, m.maxBytes, + m.retentionDuration, ) } return m @@ -580,7 +590,14 @@ func (db *DBReadOnly) Blocks() ([]BlockReader, error) { } slices.SortFunc(loadable, func(a, b *Block) int { - return int(a.Meta().MinTime - b.Meta().MinTime) + switch { + case a.Meta().MinTime < b.Meta().MinTime: + return -1 + case a.Meta().MinTime > b.Meta().MinTime: + return 1 + default: + return 0 + } }) blockMetas := make([]BlockMeta, 0, len(loadable)) @@ -649,7 +666,7 @@ func (db *DBReadOnly) Block(blockID string) (BlockReader, error) { _, err := os.Stat(filepath.Join(db.dir, blockID)) if os.IsNotExist(err) { - return nil, errors.Errorf("invalid block ID %s", blockID) + return nil, fmt.Errorf("invalid block ID %s", blockID) } block, err := OpenBlock(db.logger, filepath.Join(db.dir, blockID), nil) @@ -870,6 +887,7 @@ func open(dir string, l log.Logger, r prometheus.Registerer, opts *Options, rngs maxBytes = 0 } db.metrics.maxBytes.Set(float64(maxBytes)) + db.metrics.retentionDuration.Set((time.Duration(opts.RetentionDuration) * time.Millisecond).Seconds()) if err := db.reload(); err != nil { return nil, err @@ -1229,6 +1247,20 @@ func (db *DB) compactOOOHead(ctx context.Context) error { lastWBLFile, minOOOMmapRef := oooHead.LastWBLFile(), oooHead.LastMmapRef() if lastWBLFile != 0 || minOOOMmapRef != 0 { + if minOOOMmapRef != 0 { + // Ensure that no more queriers are created that will reference chunks we're about to garbage collect. + // truncateOOO waits for any existing queriers that reference chunks we're about to garbage collect to + // complete before running garbage collection, so we don't need to do that here. + // + // We take mtx to ensure that Querier() and ChunkQuerier() don't miss blocks: without this, they could + // capture the list of blocks before the call to reloadBlocks() above runs, but then capture + // lastGarbageCollectedMmapRef after we update it here, and therefore not query either the blocks we've just + // written or the head chunks those blocks were created from. + db.mtx.Lock() + db.lastGarbageCollectedMmapRef = minOOOMmapRef + db.mtx.Unlock() + } + if err := db.head.truncateOOO(lastWBLFile, minOOOMmapRef); err != nil { return errors.Wrap(err, "truncate ooo wbl") } @@ -1448,7 +1480,14 @@ func (db *DB) reloadBlocks() (err error) { db.metrics.blocksBytes.Set(float64(blocksSize)) slices.SortFunc(toLoad, func(a, b *Block) int { - return int(a.Meta().MinTime - b.Meta().MinTime) + switch { + case a.Meta().MinTime < b.Meta().MinTime: + return -1 + case a.Meta().MinTime > b.Meta().MinTime: + return 1 + default: + return 0 + } }) // Swap new blocks first for subsequently created readers to be seen. @@ -1518,7 +1557,14 @@ func deletableBlocks(db *DB, blocks []*Block) map[ulid.ULID]struct{} { // Sort the blocks by time - newest to oldest (largest to smallest timestamp). // This ensures that the retentions will remove the oldest blocks. slices.SortFunc(blocks, func(a, b *Block) int { - return int(b.Meta().MaxTime - a.Meta().MaxTime) + switch { + case b.Meta().MaxTime < a.Meta().MaxTime: + return -1 + case b.Meta().MaxTime > a.Meta().MaxTime: + return 1 + default: + return 0 + } }) for _, block := range blocks { @@ -1806,10 +1852,10 @@ func (db *DB) ForceHeadMMap() { // will create a new block containing all data that's currently in the memory buffer/WAL. func (db *DB) Snapshot(dir string, withHead bool) error { if dir == db.dir { - return errors.Errorf("cannot snapshot into base directory") + return fmt.Errorf("cannot snapshot into base directory") } if _, err := ulid.ParseStrict(dir); err == nil { - return errors.Errorf("dir must not be a valid ULID") + return fmt.Errorf("dir must not be a valid ULID") } db.cmtx.Lock() @@ -1841,7 +1887,7 @@ func (db *DB) Snapshot(dir string, withHead bool) error { } // Querier returns a new querier over the data partition for the given time range. -func (db *DB) Querier(mint, maxt int64) (storage.Querier, error) { +func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) { var blocks []BlockReader db.mtx.RLock() @@ -1852,11 +1898,23 @@ func (db *DB) Querier(mint, maxt int64) (storage.Querier, error) { blocks = append(blocks, b) } } - var inOrderHeadQuerier storage.Querier + + blockQueriers := make([]storage.Querier, 0, len(blocks)+2) // +2 to allow for possible in-order and OOO head queriers + + defer func() { + if err != nil { + // If we fail, all previously opened queriers must be closed. + for _, q := range blockQueriers { + // TODO(bwplotka): Handle error. + _ = q.Close() + } + } + }() + if maxt >= db.head.MinTime() { rh := NewRangeHead(db.head, mint, maxt) var err error - inOrderHeadQuerier, err = NewBlockQuerier(rh, mint, maxt) + inOrderHeadQuerier, err := NewBlockQuerier(rh, mint, maxt) if err != nil { return nil, errors.Wrapf(err, "open block querier for head %s", rh) } @@ -1878,44 +1936,40 @@ func (db *DB) Querier(mint, maxt int64) (storage.Querier, error) { return nil, errors.Wrapf(err, "open block querier for head while getting new querier %s", rh) } } + + if inOrderHeadQuerier != nil { + blockQueriers = append(blockQueriers, inOrderHeadQuerier) + } } - var outOfOrderHeadQuerier storage.Querier if overlapsClosedInterval(mint, maxt, db.head.MinOOOTime(), db.head.MaxOOOTime()) { - rh := NewOOORangeHead(db.head, mint, maxt) + rh := NewOOORangeHead(db.head, mint, maxt, db.lastGarbageCollectedMmapRef) var err error - outOfOrderHeadQuerier, err = NewBlockQuerier(rh, mint, maxt) + outOfOrderHeadQuerier, err := NewBlockQuerier(rh, mint, maxt) if err != nil { + // If NewBlockQuerier() failed, make sure to clean up the pending read created by NewOOORangeHead. + rh.isoState.Close() + return nil, errors.Wrapf(err, "open block querier for ooo head %s", rh) } + + blockQueriers = append(blockQueriers, outOfOrderHeadQuerier) } - blockQueriers := make([]storage.Querier, 0, len(blocks)) for _, b := range blocks { q, err := NewBlockQuerier(b, mint, maxt) - if err == nil { - blockQueriers = append(blockQueriers, q) - continue - } - // If we fail, all previously opened queriers must be closed. - for _, q := range blockQueriers { - // TODO(bwplotka): Handle error. - _ = q.Close() + if err != nil { + return nil, errors.Wrapf(err, "open querier for block %s", b) } - return nil, errors.Wrapf(err, "open querier for block %s", b) - } - if inOrderHeadQuerier != nil { - blockQueriers = append(blockQueriers, inOrderHeadQuerier) - } - if outOfOrderHeadQuerier != nil { - blockQueriers = append(blockQueriers, outOfOrderHeadQuerier) + blockQueriers = append(blockQueriers, q) } + return storage.NewMergeQuerier(blockQueriers, nil, storage.ChainedSeriesMerge), nil } // blockChunkQuerierForRange returns individual block chunk queriers from the persistent blocks, in-order head block, and the // out-of-order head block, overlapping with the given time range. -func (db *DB) blockChunkQuerierForRange(mint, maxt int64) ([]storage.ChunkQuerier, error) { +func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuerier, err error) { var blocks []BlockReader db.mtx.RLock() @@ -1926,11 +1980,22 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) ([]storage.ChunkQuerie blocks = append(blocks, b) } } - var inOrderHeadQuerier storage.ChunkQuerier + + blockQueriers := make([]storage.ChunkQuerier, 0, len(blocks)+2) // +2 to allow for possible in-order and OOO head queriers + + defer func() { + if err != nil { + // If we fail, all previously opened queriers must be closed. + for _, q := range blockQueriers { + // TODO(bwplotka): Handle error. + _ = q.Close() + } + } + }() + if maxt >= db.head.MinTime() { rh := NewRangeHead(db.head, mint, maxt) - var err error - inOrderHeadQuerier, err = NewBlockChunkQuerier(rh, mint, maxt) + inOrderHeadQuerier, err := NewBlockChunkQuerier(rh, mint, maxt) if err != nil { return nil, errors.Wrapf(err, "open querier for head %s", rh) } @@ -1952,37 +2017,28 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) ([]storage.ChunkQuerie return nil, errors.Wrapf(err, "open querier for head while getting new querier %s", rh) } } + + if inOrderHeadQuerier != nil { + blockQueriers = append(blockQueriers, inOrderHeadQuerier) + } } - var outOfOrderHeadQuerier storage.ChunkQuerier if overlapsClosedInterval(mint, maxt, db.head.MinOOOTime(), db.head.MaxOOOTime()) { - rh := NewOOORangeHead(db.head, mint, maxt) - var err error - outOfOrderHeadQuerier, err = NewBlockChunkQuerier(rh, mint, maxt) + rh := NewOOORangeHead(db.head, mint, maxt, db.lastGarbageCollectedMmapRef) + outOfOrderHeadQuerier, err := NewBlockChunkQuerier(rh, mint, maxt) if err != nil { return nil, errors.Wrapf(err, "open block chunk querier for ooo head %s", rh) } + + blockQueriers = append(blockQueriers, outOfOrderHeadQuerier) } - blockQueriers := make([]storage.ChunkQuerier, 0, len(blocks)) for _, b := range blocks { q, err := NewBlockChunkQuerier(b, mint, maxt) - if err == nil { - blockQueriers = append(blockQueriers, q) - continue - } - // If we fail, all previously opened queriers must be closed. - for _, q := range blockQueriers { - // TODO(bwplotka): Handle error. - _ = q.Close() + if err != nil { + return nil, errors.Wrapf(err, "open querier for block %s", b) } - return nil, errors.Wrapf(err, "open querier for block %s", b) - } - if inOrderHeadQuerier != nil { - blockQueriers = append(blockQueriers, inOrderHeadQuerier) - } - if outOfOrderHeadQuerier != nil { - blockQueriers = append(blockQueriers, outOfOrderHeadQuerier) + blockQueriers = append(blockQueriers, q) } return blockQueriers, nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go b/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go index ab97876a36..cd98fbd82f 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/encoding/encoding.go @@ -15,13 +15,14 @@ package encoding import ( "encoding/binary" + "errors" + "fmt" "hash" "hash/crc32" "math" "unsafe" "github.com/dennwc/varint" - "github.com/pkg/errors" ) var ( @@ -153,7 +154,7 @@ func NewDecbufUvarintAt(bs ByteSlice, off int, castagnoliTable *crc32.Table) Dec l, n := varint.Uvarint(b) if n <= 0 || n > binary.MaxVarintLen32 { - return Decbuf{E: errors.Errorf("invalid uvarint %d", n)} + return Decbuf{E: fmt.Errorf("invalid uvarint %d", n)} } if bs.Len() < off+n+int(l)+4 { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go b/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go index aa0a4b1b36..ff230c44b1 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/errors/errors.go @@ -25,7 +25,7 @@ import ( type multiError []error // NewMulti returns multiError with provided errors added if not nil. -func NewMulti(errs ...error) multiError { // nolint:revive +func NewMulti(errs ...error) multiError { //nolint:revive // unexported-return. m := multiError{} m.Add(errs...) return m @@ -38,7 +38,8 @@ func (es *multiError) Add(errs ...error) { if err == nil { continue } - if merr, ok := err.(nonNilMultiError); ok { + var merr nonNilMultiError + if errors.As(err, &merr) { *es = append(*es, merr.errs...) continue } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/exemplar.go b/vendor/github.com/prometheus/prometheus/tsdb/exemplar.go index 904fc7c2bd..8eaf42653c 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/exemplar.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/exemplar.go @@ -245,11 +245,26 @@ func (ce *CircularExemplarStorage) validateExemplar(key []byte, e exemplar.Exemp // Check for duplicate vs last stored exemplar for this series. // NB these are expected, and appending them is a no-op. - if ce.exemplars[idx.newest].exemplar.Equals(e) { + // For floats and classic histograms, there is only 1 exemplar per series, + // so this is sufficient. For native histograms with multiple exemplars per series, + // we have another check below. + newestExemplar := ce.exemplars[idx.newest].exemplar + if newestExemplar.Equals(e) { return storage.ErrDuplicateExemplar } - if e.Ts <= ce.exemplars[idx.newest].exemplar.Ts { + // Since during the scrape the exemplars are sorted first by timestamp, then value, then labels, + // if any of these conditions are true, we know that the exemplar is either a duplicate + // of a previous one (but not the most recent one as that is checked above) or out of order. + // We now allow exemplars with duplicate timestamps as long as they have different values and/or labels + // since that can happen for different buckets of a native histogram. + // We do not distinguish between duplicates and out of order as iterating through the exemplars + // to check for that would be expensive (versus just comparing with the most recent one) especially + // since this is run under a lock, and not worth it as we just need to return an error so we do not + // append the exemplar. + if e.Ts < newestExemplar.Ts || + (e.Ts == newestExemplar.Ts && e.Value < newestExemplar.Value) || + (e.Ts == newestExemplar.Ts && e.Value == newestExemplar.Value && e.Labels.Hash() < newestExemplar.Labels.Hash()) { if appended { ce.metrics.outOfOrderExemplars.Inc() } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go b/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go index 4dbca4f974..782ff27ec9 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/fileutil/mmap.go @@ -14,9 +14,8 @@ package fileutil import ( + "fmt" "os" - - "github.com/pkg/errors" ) type MmapFile struct { @@ -31,7 +30,7 @@ func OpenMmapFile(path string) (*MmapFile, error) { func OpenMmapFileWithSize(path string, size int) (mf *MmapFile, retErr error) { f, err := os.Open(path) if err != nil { - return nil, errors.Wrap(err, "try lock file") + return nil, fmt.Errorf("try lock file: %w", err) } defer func() { if retErr != nil { @@ -41,14 +40,14 @@ func OpenMmapFileWithSize(path string, size int) (mf *MmapFile, retErr error) { if size <= 0 { info, err := f.Stat() if err != nil { - return nil, errors.Wrap(err, "stat") + return nil, fmt.Errorf("stat: %w", err) } size = int(info.Size()) } b, err := mmap(f, size) if err != nil { - return nil, errors.Wrapf(err, "mmap, size %d", size) + return nil, fmt.Errorf("mmap, size %d: %w", size, err) } return &MmapFile{f: f, b: b}, nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate_linux.go b/vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate_linux.go index ada0462213..026c69b354 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate_linux.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/fileutil/preallocate_linux.go @@ -15,6 +15,7 @@ package fileutil import ( + "errors" "os" "syscall" ) @@ -23,10 +24,10 @@ func preallocExtend(f *os.File, sizeInBytes int64) error { // use mode = 0 to change size err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes) if err != nil { - errno, ok := err.(syscall.Errno) + var errno syscall.Errno // not supported; fallback // fallocate EINTRs frequently in some environments; fallback - if ok && (errno == syscall.ENOTSUP || errno == syscall.EINTR) { + if errors.As(err, &errno) && (errno == syscall.ENOTSUP || errno == syscall.EINTR) { return preallocExtendTrunc(f, sizeInBytes) } } @@ -37,9 +38,9 @@ func preallocFixed(f *os.File, sizeInBytes int64) error { // use mode = 1 to keep size; see FALLOC_FL_KEEP_SIZE err := syscall.Fallocate(int(f.Fd()), 1, 0, sizeInBytes) if err != nil { - errno, ok := err.(syscall.Errno) + var errno syscall.Errno // treat not supported as nil error - if ok && errno == syscall.ENOTSUP { + if errors.As(err, &errno) && errno == syscall.ENOTSUP { return nil } } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head.go b/vendor/github.com/prometheus/prometheus/tsdb/head.go index 0ea2b83853..3ff2bee716 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head.go @@ -20,6 +20,7 @@ import ( "math" "path/filepath" "runtime" + "strconv" "sync" "time" @@ -106,9 +107,12 @@ type Head struct { iso *isolation + oooIso *oooIsolation + cardinalityMutex sync.Mutex cardinalityCache *index.PostingsStats // Posting stats cache which will expire after 30sec. - lastPostingsStatsCall time.Duration // Last posting stats call (PostingsCardinalityStats()) time for caching. + cardinalityCacheKey string + lastPostingsStatsCall time.Duration // Last posting stats call (PostingsCardinalityStats()) time for caching. // chunkDiskMapper is used to write and read Head chunks to/from disk. chunkDiskMapper *chunks.ChunkDiskMapper @@ -224,11 +228,11 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal, wbl *wlog.WL, opts *Hea // even if ooo is not enabled yet. capMax := opts.OutOfOrderCapMax.Load() if capMax <= 0 || capMax > 255 { - return nil, errors.Errorf("OOOCapMax of %d is invalid. must be > 0 and <= 255", capMax) + return nil, fmt.Errorf("OOOCapMax of %d is invalid. must be > 0 and <= 255", capMax) } if opts.ChunkRange < 1 { - return nil, errors.Errorf("invalid chunk range %d", opts.ChunkRange) + return nil, fmt.Errorf("invalid chunk range %d", opts.ChunkRange) } if opts.SeriesCallback == nil { opts.SeriesCallback = &noopSeriesLifecycleCallback{} @@ -300,6 +304,7 @@ func (h *Head) resetInMemoryState() error { } h.iso = newIsolation(h.opts.IsolationDisabled) + h.oooIso = newOOOIsolation() h.exemplarMetrics = em h.exemplars = es @@ -857,7 +862,7 @@ func (h *Head) loadMmappedChunks(refSeries map[chunks.HeadSeriesRef]*memSeries) slice := mmappedChunks[seriesRef] if len(slice) > 0 && slice[len(slice)-1].maxTime >= mint { h.metrics.mmapChunkCorruptionTotal.Inc() - return errors.Errorf("out of sequence m-mapped chunk for series ref %d, last chunk: [%d, %d], new: [%d, %d]", + return fmt.Errorf("out of sequence m-mapped chunk for series ref %d, last chunk: [%d, %d], new: [%d, %d]", seriesRef, slice[len(slice)-1].minTime, slice[len(slice)-1].maxTime, mint, maxt) } slice = append(slice, &mmappedChunk{ @@ -872,7 +877,7 @@ func (h *Head) loadMmappedChunks(refSeries map[chunks.HeadSeriesRef]*memSeries) if len(ms.mmappedChunks) > 0 && ms.mmappedChunks[len(ms.mmappedChunks)-1].maxTime >= mint { h.metrics.mmapChunkCorruptionTotal.Inc() - return errors.Errorf("out of sequence m-mapped chunk for series ref %d, last chunk: [%d, %d], new: [%d, %d]", + return fmt.Errorf("out of sequence m-mapped chunk for series ref %d, last chunk: [%d, %d], new: [%d, %d]", seriesRef, ms.mmappedChunks[len(ms.mmappedChunks)-1].minTime, ms.mmappedChunks[len(ms.mmappedChunks)-1].maxTime, mint, maxt) } @@ -985,16 +990,23 @@ func (h *Head) DisableNativeHistograms() { // PostingsCardinalityStats returns highest cardinality stats by label and value names. func (h *Head) PostingsCardinalityStats(statsByLabelName string, limit int) *index.PostingsStats { + cacheKey := statsByLabelName + ";" + strconv.Itoa(limit) + h.cardinalityMutex.Lock() defer h.cardinalityMutex.Unlock() - currentTime := time.Duration(time.Now().Unix()) * time.Second - seconds := currentTime - h.lastPostingsStatsCall - if seconds > cardinalityCacheExpirationTime { + if h.cardinalityCacheKey != cacheKey { h.cardinalityCache = nil + } else { + currentTime := time.Duration(time.Now().Unix()) * time.Second + seconds := currentTime - h.lastPostingsStatsCall + if seconds > cardinalityCacheExpirationTime { + h.cardinalityCache = nil + } } if h.cardinalityCache != nil { return h.cardinalityCache } + h.cardinalityCacheKey = cacheKey h.cardinalityCache = h.postings.Stats(statsByLabelName, limit) h.lastPostingsStatsCall = time.Duration(time.Now().Unix()) * time.Second @@ -1133,6 +1145,14 @@ func (h *Head) WaitForPendingReadersInTimeRange(mint, maxt int64) { } } +// WaitForPendingReadersForOOOChunksAtOrBefore is like WaitForPendingReadersInTimeRange, except it waits for +// queries touching OOO chunks less than or equal to chunk to finish querying. +func (h *Head) WaitForPendingReadersForOOOChunksAtOrBefore(chunk chunks.ChunkDiskMapperRef) { + for h.oooIso.HasOpenReadsAtOrBefore(chunk) { + time.Sleep(500 * time.Millisecond) + } +} + // WaitForAppendersOverlapping waits for appends overlapping maxt to finish. func (h *Head) WaitForAppendersOverlapping(maxt int64) { for maxt >= h.iso.lowestAppendTime() { @@ -1271,13 +1291,19 @@ func (h *Head) truncateWAL(mint int64) error { } // truncateOOO +// - waits for any pending reads that potentially touch chunks less than or equal to newMinOOOMmapRef // - truncates the OOO WBL files whose index is strictly less than lastWBLFile. -// - garbage collects all the m-map chunks from the memory that are less than or equal to minOOOMmapRef +// - garbage collects all the m-map chunks from the memory that are less than or equal to newMinOOOMmapRef // and then deletes the series that do not have any data anymore. -func (h *Head) truncateOOO(lastWBLFile int, minOOOMmapRef chunks.ChunkDiskMapperRef) error { +// +// The caller is responsible for ensuring that no further queriers will be created that reference chunks less +// than or equal to newMinOOOMmapRef before calling truncateOOO. +func (h *Head) truncateOOO(lastWBLFile int, newMinOOOMmapRef chunks.ChunkDiskMapperRef) error { curMinOOOMmapRef := chunks.ChunkDiskMapperRef(h.minOOOMmapRef.Load()) - if minOOOMmapRef.GreaterThan(curMinOOOMmapRef) { - h.minOOOMmapRef.Store(uint64(minOOOMmapRef)) + if newMinOOOMmapRef.GreaterThan(curMinOOOMmapRef) { + h.WaitForPendingReadersForOOOChunksAtOrBefore(newMinOOOMmapRef) + h.minOOOMmapRef.Store(uint64(newMinOOOMmapRef)) + if err := h.truncateSeriesAndChunkDiskMapper("truncateOOO"); err != nil { return err } @@ -1407,11 +1433,13 @@ func (h *RangeHead) NumSeries() uint64 { return h.head.NumSeries() } +var rangeHeadULID = ulid.MustParse("0000000000XXXXXXXRANGEHEAD") + func (h *RangeHead) Meta() BlockMeta { return BlockMeta{ MinTime: h.MinTime(), MaxTime: h.MaxTime(), - ULID: h.head.Meta().ULID, + ULID: rangeHeadULID, Stats: BlockStats{ NumSeries: h.NumSeries(), }, @@ -1527,7 +1555,7 @@ func (h *Head) gc() (actualInOrderMint, minOOOTime int64, minMmapFile int) { return actualInOrderMint, minOOOTime, minMmapFile } -// Tombstones returns a new reader over the head's tombstones +// Tombstones returns a new reader over the head's tombstones. func (h *Head) Tombstones() (tombstones.Reader, error) { return h.tombstones, nil } @@ -1537,15 +1565,15 @@ func (h *Head) NumSeries() uint64 { return h.numSeries.Load() } +var headULID = ulid.MustParse("0000000000XXXXXXXXXXXXHEAD") + // Meta returns meta information about the head. // The head is dynamic so will return dynamic results. func (h *Head) Meta() BlockMeta { - var id [16]byte - copy(id[:], "______head______") return BlockMeta{ MinTime: h.MinTime(), MaxTime: h.MaxTime(), - ULID: ulid.ULID(id), + ULID: headULID, Stats: BlockStats{ NumSeries: h.NumSeries(), }, @@ -1593,9 +1621,6 @@ func (h *Head) Close() error { h.mmapHeadChunks() errs := tsdb_errors.NewMulti(h.chunkDiskMapper.Close()) - if errs.Err() == nil && h.opts.EnableMemorySnapshotOnShutdown { - errs.Add(h.performChunkSnapshot()) - } if h.wal != nil { errs.Add(h.wal.Close()) } @@ -1667,26 +1692,34 @@ func (h *Head) mmapHeadChunks() { var count int for i := 0; i < h.series.size; i++ { h.series.locks[i].RLock() - for _, all := range h.series.hashes[i] { - for _, series := range all { - series.Lock() - count += series.mmapChunks(h.chunkDiskMapper) - series.Unlock() - } + for _, series := range h.series.series[i] { + series.Lock() + count += series.mmapChunks(h.chunkDiskMapper) + series.Unlock() } h.series.locks[i].RUnlock() } h.metrics.mmapChunksTotal.Add(float64(count)) } -// seriesHashmap is a simple hashmap for memSeries by their label set. It is built -// on top of a regular hashmap and holds a slice of series to resolve hash collisions. +// seriesHashmap lets TSDB find a memSeries by its label set, via a 64-bit hash. +// There is one map for the common case where the hash value is unique, and a +// second map for the case that two series have the same hash value. +// Each series is in only one of the maps. // Its methods require the hash to be submitted with it to avoid re-computations throughout // the code. -type seriesHashmap map[uint64][]*memSeries +type seriesHashmap struct { + unique map[uint64]*memSeries + conflicts map[uint64][]*memSeries +} -func (m seriesHashmap) get(hash uint64, lset labels.Labels) *memSeries { - for _, s := range m[hash] { +func (m *seriesHashmap) get(hash uint64, lset labels.Labels) *memSeries { + if s, found := m.unique[hash]; found { + if labels.Equal(s.lset, lset) { + return s + } + } + for _, s := range m.conflicts[hash] { if labels.Equal(s.lset, lset) { return s } @@ -1694,28 +1727,50 @@ func (m seriesHashmap) get(hash uint64, lset labels.Labels) *memSeries { return nil } -func (m seriesHashmap) set(hash uint64, s *memSeries) { - l := m[hash] +func (m *seriesHashmap) set(hash uint64, s *memSeries) { + if existing, found := m.unique[hash]; !found || labels.Equal(existing.lset, s.lset) { + m.unique[hash] = s + return + } + if m.conflicts == nil { + m.conflicts = make(map[uint64][]*memSeries) + } + l := m.conflicts[hash] for i, prev := range l { if labels.Equal(prev.lset, s.lset) { l[i] = s return } } - m[hash] = append(l, s) + m.conflicts[hash] = append(l, s) } -func (m seriesHashmap) del(hash uint64, lset labels.Labels) { +func (m *seriesHashmap) del(hash uint64, lset labels.Labels) { var rem []*memSeries - for _, s := range m[hash] { - if !labels.Equal(s.lset, lset) { - rem = append(rem, s) + unique, found := m.unique[hash] + switch { + case !found: + return + case labels.Equal(unique.lset, lset): + conflicts := m.conflicts[hash] + if len(conflicts) == 0 { + delete(m.unique, hash) + return + } + rem = conflicts + default: + rem = append(rem, unique) + for _, s := range m.conflicts[hash] { + if !labels.Equal(s.lset, lset) { + rem = append(rem, s) + } } } - if len(rem) == 0 { - delete(m, hash) + m.unique[hash] = rem[0] + if len(rem) == 1 { + delete(m.conflicts, hash) } else { - m[hash] = rem + m.conflicts[hash] = rem[1:] } } @@ -1757,7 +1812,10 @@ func newStripeSeries(stripeSize int, seriesCallback SeriesLifecycleCallback) *st s.series[i] = map[chunks.HeadSeriesRef]*memSeries{} } for i := range s.hashes { - s.hashes[i] = seriesHashmap{} + s.hashes[i] = seriesHashmap{ + unique: map[uint64]*memSeries{}, + conflicts: nil, // Initialized on demand in set(). + } } return s } @@ -1777,72 +1835,76 @@ func (s *stripeSeries) gc(mint int64, minOOOMmapRef chunks.ChunkDiskMapperRef) ( deletedFromPrevStripe = 0 ) minMmapFile = math.MaxInt32 - // Run through all series and truncate old chunks. Mark those with no - // chunks left as deleted and store their ID. - for i := 0; i < s.size; i++ { - deletedForCallback := make(map[chunks.HeadSeriesRef]labels.Labels, deletedFromPrevStripe) - s.locks[i].Lock() - for hash, all := range s.hashes[i] { - for _, series := range all { - series.Lock() - rmChunks += series.truncateChunksBefore(mint, minOOOMmapRef) - - if len(series.mmappedChunks) > 0 { - seq, _ := series.mmappedChunks[0].ref.Unpack() - if seq < minMmapFile { - minMmapFile = seq - } - } - if series.ooo != nil && len(series.ooo.oooMmappedChunks) > 0 { - seq, _ := series.ooo.oooMmappedChunks[0].ref.Unpack() - if seq < minMmapFile { - minMmapFile = seq - } - for _, ch := range series.ooo.oooMmappedChunks { - if ch.minTime < minOOOTime { - minOOOTime = ch.minTime - } - } - } - if series.ooo != nil && series.ooo.oooHeadChunk != nil { - if series.ooo.oooHeadChunk.minTime < minOOOTime { - minOOOTime = series.ooo.oooHeadChunk.minTime - } - } - if len(series.mmappedChunks) > 0 || series.headChunks != nil || series.pendingCommit || - (series.ooo != nil && (len(series.ooo.oooMmappedChunks) > 0 || series.ooo.oooHeadChunk != nil)) { - seriesMint := series.minTime() - if seriesMint < actualMint { - actualMint = seriesMint - } - series.Unlock() - continue - } + // For one series, truncate old chunks and check if any chunks left. If not, mark as deleted and collect the ID. + check := func(hashShard int, hash uint64, series *memSeries, deletedForCallback map[chunks.HeadSeriesRef]labels.Labels) { + series.Lock() + defer series.Unlock() - // The series is gone entirely. We need to keep the series lock - // and make sure we have acquired the stripe locks for hash and ID of the - // series alike. - // If we don't hold them all, there's a very small chance that a series receives - // samples again while we are half-way into deleting it. - j := int(series.ref) & (s.size - 1) + rmChunks += series.truncateChunksBefore(mint, minOOOMmapRef) - if i != j { - s.locks[j].Lock() + if len(series.mmappedChunks) > 0 { + seq, _ := series.mmappedChunks[0].ref.Unpack() + if seq < minMmapFile { + minMmapFile = seq + } + } + if series.ooo != nil && len(series.ooo.oooMmappedChunks) > 0 { + seq, _ := series.ooo.oooMmappedChunks[0].ref.Unpack() + if seq < minMmapFile { + minMmapFile = seq + } + for _, ch := range series.ooo.oooMmappedChunks { + if ch.minTime < minOOOTime { + minOOOTime = ch.minTime } + } + } + if series.ooo != nil && series.ooo.oooHeadChunk != nil { + if series.ooo.oooHeadChunk.minTime < minOOOTime { + minOOOTime = series.ooo.oooHeadChunk.minTime + } + } + if len(series.mmappedChunks) > 0 || series.headChunks != nil || series.pendingCommit || + (series.ooo != nil && (len(series.ooo.oooMmappedChunks) > 0 || series.ooo.oooHeadChunk != nil)) { + seriesMint := series.minTime() + if seriesMint < actualMint { + actualMint = seriesMint + } + return + } + // The series is gone entirely. We need to keep the series lock + // and make sure we have acquired the stripe locks for hash and ID of the + // series alike. + // If we don't hold them all, there's a very small chance that a series receives + // samples again while we are half-way into deleting it. + refShard := int(series.ref) & (s.size - 1) + if hashShard != refShard { + s.locks[refShard].Lock() + defer s.locks[refShard].Unlock() + } - deleted[storage.SeriesRef(series.ref)] = struct{}{} - s.hashes[i].del(hash, series.lset) - delete(s.series[j], series.ref) - deletedForCallback[series.ref] = series.lset + deleted[storage.SeriesRef(series.ref)] = struct{}{} + s.hashes[hashShard].del(hash, series.lset) + delete(s.series[refShard], series.ref) + deletedForCallback[series.ref] = series.lset + } - if i != j { - s.locks[j].Unlock() - } + // Run through all series shard by shard, checking which should be deleted. + for i := 0; i < s.size; i++ { + deletedForCallback := make(map[chunks.HeadSeriesRef]labels.Labels, deletedFromPrevStripe) + s.locks[i].Lock() - series.Unlock() + // Delete conflicts first so seriesHashmap.del doesn't move them to the `unique` field, + // after deleting `unique`. + for hash, all := range s.hashes[i].conflicts { + for _, series := range all { + check(i, hash, series, deletedForCallback) } } + for hash, series := range s.hashes[i].unique { + check(i, hash, series, deletedForCallback) + } s.locks[i].Unlock() @@ -2171,7 +2233,7 @@ func overlapsClosedInterval(mint1, maxt1, mint2, maxt2 int64) bool { return mint1 <= maxt2 && mint2 <= maxt1 } -// mmappedChunk describes a head chunk on disk that has been mmapped +// mmappedChunk describes a head chunk on disk that has been mmapped. type mmappedChunk struct { ref chunks.ChunkDiskMapperRef numSamples uint16 diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head_append.go b/vendor/github.com/prometheus/prometheus/tsdb/head_append.go index d1f4d3035e..be53a4f3f6 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head_append.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head_append.go @@ -521,13 +521,13 @@ func (a *headAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels } if h != nil { - if err := ValidateHistogram(h); err != nil { + if err := h.Validate(); err != nil { return 0, err } } if fh != nil { - if err := ValidateFloatHistogram(fh); err != nil { + if err := fh.Validate(); err != nil { return 0, err } } @@ -642,103 +642,6 @@ func (a *headAppender) UpdateMetadata(ref storage.SeriesRef, lset labels.Labels, return ref, nil } -func ValidateHistogram(h *histogram.Histogram) error { - if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil { - return errors.Wrap(err, "negative side") - } - if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil { - return errors.Wrap(err, "positive side") - } - var nCount, pCount uint64 - err := checkHistogramBuckets(h.NegativeBuckets, &nCount, true) - if err != nil { - return errors.Wrap(err, "negative side") - } - err = checkHistogramBuckets(h.PositiveBuckets, &pCount, true) - if err != nil { - return errors.Wrap(err, "positive side") - } - - if c := nCount + pCount + h.ZeroCount; c > h.Count { - return errors.Wrap( - storage.ErrHistogramCountNotBigEnough, - fmt.Sprintf("%d observations found in buckets, but the Count field is %d", c, h.Count), - ) - } - - return nil -} - -func ValidateFloatHistogram(h *histogram.FloatHistogram) error { - if err := checkHistogramSpans(h.NegativeSpans, len(h.NegativeBuckets)); err != nil { - return errors.Wrap(err, "negative side") - } - if err := checkHistogramSpans(h.PositiveSpans, len(h.PositiveBuckets)); err != nil { - return errors.Wrap(err, "positive side") - } - var nCount, pCount float64 - err := checkHistogramBuckets(h.NegativeBuckets, &nCount, false) - if err != nil { - return errors.Wrap(err, "negative side") - } - err = checkHistogramBuckets(h.PositiveBuckets, &pCount, false) - if err != nil { - return errors.Wrap(err, "positive side") - } - - // We do not check for h.Count being at least as large as the sum of the - // counts in the buckets because floating point precision issues can - // create false positives here. - - return nil -} - -func checkHistogramSpans(spans []histogram.Span, numBuckets int) error { - var spanBuckets int - for n, span := range spans { - if n > 0 && span.Offset < 0 { - return errors.Wrap( - storage.ErrHistogramSpanNegativeOffset, - fmt.Sprintf("span number %d with offset %d", n+1, span.Offset), - ) - } - spanBuckets += int(span.Length) - } - if spanBuckets != numBuckets { - return errors.Wrap( - storage.ErrHistogramSpansBucketsMismatch, - fmt.Sprintf("spans need %d buckets, have %d buckets", spanBuckets, numBuckets), - ) - } - return nil -} - -func checkHistogramBuckets[BC histogram.BucketCount, IBC histogram.InternalBucketCount](buckets []IBC, count *BC, deltas bool) error { - if len(buckets) == 0 { - return nil - } - - var last IBC - for i := 0; i < len(buckets); i++ { - var c IBC - if deltas { - c = last + buckets[i] - } else { - c = buckets[i] - } - if c < 0 { - return errors.Wrap( - storage.ErrHistogramNegativeBucketCount, - fmt.Sprintf("bucket number %d has observation count of %v", i+1, c), - ) - } - last = c - *count += BC(c) - } - - return nil -} - var _ storage.GetRef = &headAppender{} func (a *headAppender) GetRef(lset labels.Labels, hash uint64) (storage.SeriesRef, labels.Labels) { @@ -786,14 +689,6 @@ func (a *headAppender) log() error { return errors.Wrap(err, "log samples") } } - if len(a.exemplars) > 0 { - rec = enc.Exemplars(exemplarsForEncoding(a.exemplars), buf) - buf = rec[:0] - - if err := a.head.wal.Log(rec); err != nil { - return errors.Wrap(err, "log exemplars") - } - } if len(a.histograms) > 0 { rec = enc.HistogramSamples(a.histograms, buf) buf = rec[:0] @@ -808,6 +703,18 @@ func (a *headAppender) log() error { return errors.Wrap(err, "log float histograms") } } + // Exemplars should be logged after samples (float/native histogram/etc), + // otherwise it might happen that we send the exemplars in a remote write + // batch before the samples, which in turn means the exemplar is rejected + // for missing series, since series are created due to samples. + if len(a.exemplars) > 0 { + rec = enc.Exemplars(exemplarsForEncoding(a.exemplars), buf) + buf = rec[:0] + + if err := a.head.wal.Log(rec); err != nil { + return errors.Wrap(err, "log exemplars") + } + } return nil } @@ -844,6 +751,12 @@ func (a *headAppender) Commit() (err error) { // No errors logging to WAL, so pass the exemplars along to the in memory storage. for _, e := range a.exemplars { s := a.head.series.getByID(chunks.HeadSeriesRef(e.ref)) + if s == nil { + // This is very unlikely to happen, but we have seen it in the wild. + // It means that the series was truncated between AppendExemplar and Commit. + // See TestHeadCompactionWhileAppendAndCommitExemplar. + continue + } // We don't instrument exemplar appends here, all is instrumented by storage. if err := a.head.exemplars.AddExemplar(s.lset, e.exemplar); err != nil { if err == storage.ErrOutOfOrderExemplar { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head_read.go b/vendor/github.com/prometheus/prometheus/tsdb/head_read.go index 6964d5472b..35ef26a58a 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head_read.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head_read.go @@ -202,7 +202,7 @@ func (h *headIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchB // headChunkID returns the HeadChunkID referred to by the given position. // * 0 <= pos < len(s.mmappedChunks) refer to s.mmappedChunks[pos] -// * pos >= len(s.mmappedChunks) refers to s.headChunks linked list +// * pos >= len(s.mmappedChunks) refers to s.headChunks linked list. func (s *memSeries) headChunkID(pos int) chunks.HeadChunkID { return chunks.HeadChunkID(pos) + s.firstChunkID } @@ -289,10 +289,10 @@ func (h *headChunkReader) Close() error { return nil } -// Chunk returns the chunk for the reference number. -func (h *headChunkReader) Chunk(meta chunks.Meta) (chunkenc.Chunk, error) { +// ChunkOrIterable returns the chunk for the reference number. +func (h *headChunkReader) ChunkOrIterable(meta chunks.Meta) (chunkenc.Chunk, chunkenc.Iterable, error) { chk, _, err := h.chunk(meta, false) - return chk, err + return chk, nil, err } // ChunkWithCopy returns the chunk for the reference number. @@ -416,13 +416,13 @@ func (s *memSeries) chunk(id chunks.HeadChunkID, chunkDiskMapper *chunks.ChunkDi return elem, true, offset == 0, nil } -// oooMergedChunk returns the requested chunk based on the given chunks.Meta -// reference from memory or by m-mapping it from the disk. The returned chunk -// might be a merge of all the overlapping chunks, if any, amongst all the -// chunks in the OOOHead. +// oooMergedChunks return an iterable over one or more OOO chunks for the given +// chunks.Meta reference from memory or by m-mapping it from the disk. The +// returned iterable will be a merge of all the overlapping chunks, if any, +// amongst all the chunks in the OOOHead. // This function is not thread safe unless the caller holds a lock. // The caller must ensure that s.ooo is not nil. -func (s *memSeries) oooMergedChunk(meta chunks.Meta, cdm *chunks.ChunkDiskMapper, mint, maxt int64) (chunk *mergedOOOChunks, err error) { +func (s *memSeries) oooMergedChunks(meta chunks.Meta, cdm *chunks.ChunkDiskMapper, mint, maxt int64) (*mergedOOOChunks, error) { _, cid := chunks.HeadChunkRef(meta.Ref).Unpack() // ix represents the index of chunk in the s.mmappedChunks slice. The chunk meta's are @@ -499,11 +499,13 @@ func (s *memSeries) oooMergedChunk(meta chunks.Meta, cdm *chunks.ChunkDiskMapper mc := &mergedOOOChunks{} absoluteMax := int64(math.MinInt64) for _, c := range tmpChks { - if c.meta.Ref != meta.Ref && (len(mc.chunks) == 0 || c.meta.MinTime > absoluteMax) { + if c.meta.Ref != meta.Ref && (len(mc.chunkIterables) == 0 || c.meta.MinTime > absoluteMax) { continue } + var iterable chunkenc.Iterable if c.meta.Ref == oooHeadRef { var xor *chunkenc.XORChunk + var err error // If head chunk min and max time match the meta OOO markers // that means that the chunk has not expanded so we can append // it as it is. @@ -516,7 +518,7 @@ func (s *memSeries) oooMergedChunk(meta chunks.Meta, cdm *chunks.ChunkDiskMapper if err != nil { return nil, errors.Wrap(err, "failed to convert ooo head chunk to xor chunk") } - c.meta.Chunk = xor + iterable = xor } else { chk, err := cdm.Chunk(c.ref) if err != nil { @@ -531,12 +533,12 @@ func (s *memSeries) oooMergedChunk(meta chunks.Meta, cdm *chunks.ChunkDiskMapper // wrap the chunk within a chunk that doesnt allows us to iterate // through samples out of the OOOLastMinT and OOOLastMaxT // markers. - c.meta.Chunk = boundedChunk{chk, meta.OOOLastMinTime, meta.OOOLastMaxTime} + iterable = boundedIterable{chk, meta.OOOLastMinTime, meta.OOOLastMaxTime} } else { - c.meta.Chunk = chk + iterable = chk } } - mc.chunks = append(mc.chunks, c.meta) + mc.chunkIterables = append(mc.chunkIterables, iterable) if c.meta.MaxTime > absoluteMax { absoluteMax = c.meta.MaxTime } @@ -545,77 +547,30 @@ func (s *memSeries) oooMergedChunk(meta chunks.Meta, cdm *chunks.ChunkDiskMapper return mc, nil } -var _ chunkenc.Chunk = &mergedOOOChunks{} +var _ chunkenc.Iterable = &mergedOOOChunks{} -// mergedOOOChunks holds the list of overlapping chunks. This struct satisfies -// chunkenc.Chunk. +// mergedOOOChunks holds the list of iterables for overlapping chunks. type mergedOOOChunks struct { - chunks []chunks.Meta -} - -// Bytes is a very expensive method because its calling the iterator of all the -// chunks in the mergedOOOChunk and building a new chunk with the samples. -func (o mergedOOOChunks) Bytes() []byte { - xc := chunkenc.NewXORChunk() - app, err := xc.Appender() - if err != nil { - panic(err) - } - it := o.Iterator(nil) - for it.Next() == chunkenc.ValFloat { - t, v := it.At() - app.Append(t, v) - } - - return xc.Bytes() -} - -func (o mergedOOOChunks) Encoding() chunkenc.Encoding { - return chunkenc.EncXOR -} - -func (o mergedOOOChunks) Appender() (chunkenc.Appender, error) { - return nil, errors.New("can't append to mergedOOOChunks") + chunkIterables []chunkenc.Iterable } func (o mergedOOOChunks) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { - return storage.ChainSampleIteratorFromMetas(iterator, o.chunks) + return storage.ChainSampleIteratorFromIterables(iterator, o.chunkIterables) } -func (o mergedOOOChunks) NumSamples() int { - samples := 0 - for _, c := range o.chunks { - samples += c.Chunk.NumSamples() - } - return samples -} - -func (o mergedOOOChunks) Compact() {} +var _ chunkenc.Iterable = &boundedIterable{} -var _ chunkenc.Chunk = &boundedChunk{} - -// boundedChunk is an implementation of chunkenc.Chunk that uses a +// boundedIterable is an implementation of chunkenc.Iterable that uses a // boundedIterator that only iterates through samples which timestamps are -// >= minT and <= maxT -type boundedChunk struct { - chunkenc.Chunk - minT int64 - maxT int64 -} - -func (b boundedChunk) Bytes() []byte { - xor := chunkenc.NewXORChunk() - a, _ := xor.Appender() - it := b.Iterator(nil) - for it.Next() == chunkenc.ValFloat { - t, v := it.At() - a.Append(t, v) - } - return xor.Bytes() +// >= minT and <= maxT. +type boundedIterable struct { + chunk chunkenc.Chunk + minT int64 + maxT int64 } -func (b boundedChunk) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { - it := b.Chunk.Iterator(iterator) +func (b boundedIterable) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { + it := b.chunk.Iterator(iterator) if it == nil { panic("iterator shouldn't be nil") } @@ -625,7 +580,7 @@ func (b boundedChunk) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { var _ chunkenc.Iterator = &boundedIterator{} // boundedIterator is an implementation of Iterator that only iterates through -// samples which timestamps are >= minT and <= maxT +// samples which timestamps are >= minT and <= maxT. type boundedIterator struct { chunkenc.Iterator minT int64 @@ -671,7 +626,7 @@ func (b boundedIterator) Seek(t int64) chunkenc.ValueType { return b.Iterator.Seek(t) } -// safeHeadChunk makes sure that the chunk can be accessed without a race condition +// safeHeadChunk makes sure that the chunk can be accessed without a race condition. type safeHeadChunk struct { chunkenc.Chunk s *memSeries diff --git a/vendor/github.com/prometheus/prometheus/tsdb/head_wal.go b/vendor/github.com/prometheus/prometheus/tsdb/head_wal.go index d6780c021a..07fa8280ca 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/head_wal.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/head_wal.go @@ -11,7 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive // Many legitimately empty blocks in this file. package tsdb import ( @@ -971,7 +970,7 @@ func decodeSeriesFromChunkSnapshot(d *record.Decoder, b []byte) (csr chunkSnapsh dec := encoding.Decbuf{B: b} if flag := dec.Byte(); flag != chunkSnapshotRecordTypeSeries { - return csr, errors.Errorf("invalid record type %x", flag) + return csr, fmt.Errorf("invalid record type %x", flag) } csr.ref = chunks.HeadSeriesRef(dec.Be64()) @@ -1019,7 +1018,7 @@ func decodeSeriesFromChunkSnapshot(d *record.Decoder, b []byte) (csr chunkSnapsh err = dec.Err() if err != nil && len(dec.B) > 0 { - err = errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + err = fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return @@ -1042,7 +1041,7 @@ func decodeTombstonesSnapshotRecord(b []byte) (tombstones.Reader, error) { dec := encoding.Decbuf{B: b} if flag := dec.Byte(); flag != chunkSnapshotRecordTypeTombstones { - return nil, errors.Errorf("invalid record type %x", flag) + return nil, fmt.Errorf("invalid record type %x", flag) } tr, err := tombstones.Decode(dec.UvarintBytes()) @@ -1255,7 +1254,7 @@ func LastChunkSnapshot(dir string) (string, int, int, error) { continue } if !fi.IsDir() { - return "", 0, 0, errors.Errorf("chunk snapshot %s is not a directory", fi.Name()) + return "", 0, 0, fmt.Errorf("chunk snapshot %s is not a directory", fi.Name()) } splits := strings.Split(fi.Name()[len(chunkSnapshotPrefix):], ".") @@ -1493,7 +1492,7 @@ Outer: default: // This is a record type we don't understand. It is either and old format from earlier versions, // or a new format and the code was rolled back to old version. - loopErr = errors.Errorf("unsupported snapshot record type 0b%b", rec[0]) + loopErr = fmt.Errorf("unsupported snapshot record type 0b%b", rec[0]) break Outer } } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/index/index.go b/vendor/github.com/prometheus/prometheus/tsdb/index/index.go index ccf06e8eab..44ee66386f 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/index/index.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/index/index.go @@ -28,7 +28,6 @@ import ( "sort" "unsafe" - "github.com/pkg/errors" "golang.org/x/exp/slices" "github.com/prometheus/prometheus/model/labels" @@ -107,8 +106,8 @@ func newCRC32() hash.Hash32 { type symbolCacheEntry struct { index uint32 - lastValue string lastValueIndex uint32 + lastValue string } // Writer implements the IndexWriter interface for the standard @@ -172,7 +171,7 @@ func NewTOCFromByteSlice(bs ByteSlice) (*TOC, error) { d := encoding.Decbuf{B: b[:len(b)-4]} if d.Crc32(castagnoliTable) != expCRC { - return nil, errors.Wrap(encoding.ErrInvalidChecksum, "read TOC") + return nil, fmt.Errorf("read TOC: %w", encoding.ErrInvalidChecksum) } toc := &TOC{ @@ -197,7 +196,7 @@ func NewWriter(ctx context.Context, fn string) (*Writer, error) { defer df.Close() // Close for platform windows. if err := os.RemoveAll(fn); err != nil { - return nil, errors.Wrap(err, "remove any existing index at path") + return nil, fmt.Errorf("remove any existing index at path: %w", err) } // Main index file we are building. @@ -216,7 +215,7 @@ func NewWriter(ctx context.Context, fn string) (*Writer, error) { return nil, err } if err := df.Sync(); err != nil { - return nil, errors.Wrap(err, "sync dir") + return nil, fmt.Errorf("sync dir: %w", err) } iw := &Writer{ @@ -288,7 +287,7 @@ func (fw *FileWriter) Write(bufs ...[]byte) error { // Once we move to compressed/varint representations in those areas, this limitation // can be lifted. if fw.pos > 16*math.MaxUint32 { - return errors.Errorf("%q exceeding max size of 64GiB", fw.name) + return fmt.Errorf("%q exceeding max size of 64GiB", fw.name) } } return nil @@ -315,7 +314,7 @@ func (fw *FileWriter) AddPadding(size int) error { p = uint64(size) - p if err := fw.Write(make([]byte, p)); err != nil { - return errors.Wrap(err, "add padding") + return fmt.Errorf("add padding: %w", err) } return nil } @@ -353,7 +352,7 @@ func (w *Writer) ensureStage(s indexWriterStage) error { } } if w.stage > s { - return errors.Errorf("invalid stage %q, currently at %q", s, w.stage) + return fmt.Errorf("invalid stage %q, currently at %q", s, w.stage) } // Mark start of sections in table of contents. @@ -417,20 +416,20 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... return err } if labels.Compare(lset, w.lastSeries) <= 0 { - return errors.Errorf("out-of-order series added with label set %q", lset) + return fmt.Errorf("out-of-order series added with label set %q", lset) } if ref < w.lastRef && !w.lastSeries.IsEmpty() { - return errors.Errorf("series with reference greater than %d already added", ref) + return fmt.Errorf("series with reference greater than %d already added", ref) } // We add padding to 16 bytes to increase the addressable space we get through 4 byte // series references. if err := w.addPadding(16); err != nil { - return errors.Errorf("failed to write padding bytes: %v", err) + return fmt.Errorf("failed to write padding bytes: %v", err) } if w.f.pos%16 != 0 { - return errors.Errorf("series write not 16-byte aligned at %d", w.f.pos) + return fmt.Errorf("series write not 16-byte aligned at %d", w.f.pos) } w.buf2.Reset() @@ -443,7 +442,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... if !ok { nameIndex, err = w.symbols.ReverseLookup(l.Name) if err != nil { - return errors.Errorf("symbol entry for %q does not exist, %v", l.Name, err) + return fmt.Errorf("symbol entry for %q does not exist, %v", l.Name, err) } } w.labelNames[l.Name]++ @@ -453,12 +452,12 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... if !ok || cacheEntry.lastValue != l.Value { valueIndex, err = w.symbols.ReverseLookup(l.Value) if err != nil { - return errors.Errorf("symbol entry for %q does not exist, %v", l.Value, err) + return fmt.Errorf("symbol entry for %q does not exist, %v", l.Value, err) } w.symbolCache[l.Name] = symbolCacheEntry{ index: nameIndex, - lastValue: l.Value, lastValueIndex: valueIndex, + lastValue: l.Value, } } w.buf2.PutUvarint32(valueIndex) @@ -493,7 +492,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ... w.buf2.PutHash(w.crc32) if err := w.write(w.buf1.Get(), w.buf2.Get()); err != nil { - return errors.Wrap(err, "write series data") + return fmt.Errorf("write series data: %w", err) } w.lastSeries.CopyFrom(lset) @@ -514,7 +513,7 @@ func (w *Writer) AddSymbol(sym string) error { return err } if w.numSymbols != 0 && sym <= w.lastSymbol { - return errors.Errorf("symbol %q out-of-order", sym) + return fmt.Errorf("symbol %q out-of-order", sym) } w.lastSymbol = sym w.numSymbols++ @@ -527,7 +526,7 @@ func (w *Writer) finishSymbols() error { symbolTableSize := w.f.pos - w.toc.Symbols - 4 // The symbol table's part is 4 bytes. So the total symbol table size must be less than or equal to 2^32-1 if symbolTableSize > math.MaxUint32 { - return errors.Errorf("symbol table size exceeds %d bytes: %d", uint32(math.MaxUint32), symbolTableSize) + return fmt.Errorf("symbol table size exceeds %d bytes: %d", uint32(math.MaxUint32), symbolTableSize) } // Write out the length and symbol count. @@ -563,7 +562,7 @@ func (w *Writer) finishSymbols() error { // Load in the symbol table efficiently for the rest of the index writing. w.symbols, err = NewSymbols(realByteSlice(w.symbolFile.Bytes()), FormatV2, int(w.toc.Symbols)) if err != nil { - return errors.Wrap(err, "read symbols") + return fmt.Errorf("read symbols: %w", err) } return nil } @@ -660,7 +659,7 @@ func (w *Writer) writeLabelIndex(name string, values []uint32) error { w.buf1.Reset() l := w.f.pos - startPos - 4 if l > math.MaxUint32 { - return errors.Errorf("label index size exceeds 4 bytes: %d", l) + return fmt.Errorf("label index size exceeds 4 bytes: %d", l) } w.buf1.PutBE32int(int(l)) if err := w.writeAt(w.buf1.Get(), startPos); err != nil { @@ -704,7 +703,7 @@ func (w *Writer) writeLabelIndexesOffsetTable() error { w.buf1.Reset() l := w.f.pos - startPos - 4 if l > math.MaxUint32 { - return errors.Errorf("label indexes offset table size exceeds 4 bytes: %d", l) + return fmt.Errorf("label indexes offset table size exceeds 4 bytes: %d", l) } w.buf1.PutBE32int(int(l)) if err := w.writeAt(w.buf1.Get(), startPos); err != nil { @@ -785,7 +784,7 @@ func (w *Writer) writePostingsOffsetTable() error { w.buf1.Reset() l := w.f.pos - startPos - 4 if l > math.MaxUint32 { - return errors.Errorf("postings offset table size exceeds 4 bytes: %d", l) + return fmt.Errorf("postings offset table size exceeds 4 bytes: %d", l) } w.buf1.PutBE32int(int(l)) if err := w.writeAt(w.buf1.Get(), startPos); err != nil { @@ -839,7 +838,7 @@ func (w *Writer) writePostingsToTmpFiles() error { d.ConsumePadding() startPos := w.toc.LabelIndices - uint64(d.Len()) if startPos%16 != 0 { - return errors.Errorf("series not 16-byte aligned at %d", startPos) + return fmt.Errorf("series not 16-byte aligned at %d", startPos) } offsets = append(offsets, uint32(startPos/16)) // Skip to next series. @@ -923,7 +922,7 @@ func (w *Writer) writePostingsToTmpFiles() error { // Symbol numbers are in order, so the strings will also be in order. slices.Sort(values) for _, v := range values { - value, err := w.symbols.Lookup(w.ctx, v) + value, err := w.symbols.Lookup(v) if err != nil { return err } @@ -964,7 +963,7 @@ func (w *Writer) writePosting(name, value string, offs []uint32) error { for _, off := range offs { if off > (1<<32)-1 { - return errors.Errorf("series offset %d exceeds 4 bytes", off) + return fmt.Errorf("series offset %d exceeds 4 bytes", off) } w.buf1.PutBE32(off) } @@ -973,7 +972,7 @@ func (w *Writer) writePosting(name, value string, offs []uint32) error { l := w.buf1.Len() // We convert to uint to make code compile on 32-bit systems, as math.MaxUint32 doesn't fit into int there. if uint(l) > math.MaxUint32 { - return errors.Errorf("posting size exceeds 4 bytes: %d", l) + return fmt.Errorf("posting size exceeds 4 bytes: %d", l) } w.buf2.PutBE32int(l) w.buf1.PutHash(w.crc32) @@ -1000,7 +999,7 @@ func (w *Writer) writePostings() error { return err } if uint64(n) != w.fP.pos { - return errors.Errorf("wrote %d bytes to posting temporary file, but only read back %d", w.fP.pos, n) + return fmt.Errorf("wrote %d bytes to posting temporary file, but only read back %d", w.fP.pos, n) } w.f.pos += uint64(n) @@ -1135,26 +1134,26 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) { // Verify header. if r.b.Len() < HeaderLen { - return nil, errors.Wrap(encoding.ErrInvalidSize, "index header") + return nil, fmt.Errorf("index header: %w", encoding.ErrInvalidSize) } if m := binary.BigEndian.Uint32(r.b.Range(0, 4)); m != MagicIndex { - return nil, errors.Errorf("invalid magic number %x", m) + return nil, fmt.Errorf("invalid magic number %x", m) } r.version = int(r.b.Range(4, 5)[0]) if r.version != FormatV1 && r.version != FormatV2 { - return nil, errors.Errorf("unknown index file version %d", r.version) + return nil, fmt.Errorf("unknown index file version %d", r.version) } var err error r.toc, err = NewTOCFromByteSlice(b) if err != nil { - return nil, errors.Wrap(err, "read TOC") + return nil, fmt.Errorf("read TOC: %w", err) } r.symbols, err = NewSymbols(r.b, r.version, int(r.toc.Symbols)) if err != nil { - return nil, errors.Wrap(err, "read symbols") + return nil, fmt.Errorf("read symbols: %w", err) } if r.version == FormatV1 { @@ -1169,7 +1168,7 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) { r.postingsV1[string(name)][string(value)] = off return nil }); err != nil { - return nil, errors.Wrap(err, "read postings table") + return nil, fmt.Errorf("read postings table: %w", err) } } else { var lastName, lastValue []byte @@ -1197,7 +1196,7 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) { valueCount++ return nil }); err != nil { - return nil, errors.Wrap(err, "read postings table") + return nil, fmt.Errorf("read postings table: %w", err) } if lastName != nil { r.postings[string(lastName)] = append(r.postings[string(lastName)], postingOffset{value: string(lastValue), off: lastOff}) @@ -1217,7 +1216,7 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) { } off, err := r.symbols.ReverseLookup(k) if err != nil { - return nil, errors.Wrap(err, "reverse symbol lookup") + return nil, fmt.Errorf("reverse symbol lookup: %w", err) } r.nameSymbols[off] = k } @@ -1252,7 +1251,7 @@ func (r *Reader) PostingsRanges() (map[labels.Label]Range, error) { } return nil }); err != nil { - return nil, errors.Wrap(err, "read postings table") + return nil, fmt.Errorf("read postings table: %w", err) } return m, nil } @@ -1295,21 +1294,18 @@ func NewSymbols(bs ByteSlice, version, off int) (*Symbols, error) { return s, nil } -func (s Symbols) Lookup(ctx context.Context, o uint32) (string, error) { +func (s Symbols) Lookup(o uint32) (string, error) { d := encoding.Decbuf{ B: s.bs.Range(0, s.bs.Len()), } if s.version == FormatV2 { if int(o) >= s.seen { - return "", errors.Errorf("unknown symbol offset %d", o) + return "", fmt.Errorf("unknown symbol offset %d", o) } d.Skip(s.offsets[int(o/symbolFactor)]) // Walk until we find the one we want. for i := o - (o / symbolFactor * symbolFactor); i > 0; i-- { - if ctx.Err() != nil { - return "", ctx.Err() - } d.UvarintBytes() } } else { @@ -1324,7 +1320,7 @@ func (s Symbols) Lookup(ctx context.Context, o uint32) (string, error) { func (s Symbols) ReverseLookup(sym string) (uint32, error) { if len(s.offsets) == 0 { - return 0, errors.Errorf("unknown symbol %q - no symbols", sym) + return 0, fmt.Errorf("unknown symbol %q - no symbols", sym) } i := sort.Search(len(s.offsets), func(i int) bool { // Any decoding errors here will be lost, however @@ -1357,7 +1353,7 @@ func (s Symbols) ReverseLookup(sym string) (uint32, error) { return 0, d.Err() } if lastSymbol != sym { - return 0, errors.Errorf("unknown symbol %q", sym) + return 0, fmt.Errorf("unknown symbol %q", sym) } if s.version == FormatV2 { return uint32(res), nil @@ -1416,7 +1412,7 @@ func ReadPostingsOffsetTable(bs ByteSlice, off uint64, f func(name, value []byte offsetPos := startLen - d.Len() if keyCount := d.Uvarint(); keyCount != 2 { - return errors.Errorf("unexpected number of keys for postings offset table %d", keyCount) + return fmt.Errorf("unexpected number of keys for postings offset table %d", keyCount) } name := d.UvarintBytes() value := d.UvarintBytes() @@ -1441,7 +1437,7 @@ func (r *Reader) lookupSymbol(ctx context.Context, o uint32) (string, error) { if s, ok := r.nameSymbols[o]; ok { return s, nil } - return r.symbols.Lookup(ctx, o) + return r.symbols.Lookup(o) } // Symbols returns an iterator over the symbols that exist within the index. @@ -1468,10 +1464,10 @@ func (r *Reader) SortedLabelValues(ctx context.Context, name string, matchers .. // LabelValues returns value tuples that exist for the given label name. // It is not safe to use the return value beyond the lifetime of the byte slice // passed into the Reader. -// TODO(replay): Support filtering by matchers +// TODO(replay): Support filtering by matchers. func (r *Reader) LabelValues(ctx context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { if len(matchers) > 0 { - return nil, errors.Errorf("matchers parameter is not implemented: %+v", matchers) + return nil, fmt.Errorf("matchers parameter is not implemented: %+v", matchers) } if r.version == FormatV1 { @@ -1519,7 +1515,7 @@ func (r *Reader) LabelValues(ctx context.Context, name string, matchers ...*labe d.Uvarint64() // Offset. } if d.Err() != nil { - return nil, errors.Wrap(d.Err(), "get postings offset entry") + return nil, fmt.Errorf("get postings offset entry: %w", d.Err()) } return values, ctx.Err() @@ -1545,12 +1541,12 @@ func (r *Reader) LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([ d := encoding.NewDecbufUvarintAt(r.b, int(offset), castagnoliTable) buf := d.Get() if d.Err() != nil { - return nil, errors.Wrap(d.Err(), "get buffer for series") + return nil, fmt.Errorf("get buffer for series: %w", d.Err()) } offsets, err := r.dec.LabelNamesOffsetsFor(buf) if err != nil { - return nil, errors.Wrap(err, "get label name offsets") + return nil, fmt.Errorf("get label name offsets: %w", err) } for _, off := range offsets { offsetsMap[off] = struct{}{} @@ -1562,7 +1558,7 @@ func (r *Reader) LabelNamesFor(ctx context.Context, ids ...storage.SeriesRef) ([ for off := range offsetsMap { name, err := r.lookupSymbol(ctx, off) if err != nil { - return nil, errors.Wrap(err, "lookup symbol in LabelNamesFor") + return nil, fmt.Errorf("lookup symbol in LabelNamesFor: %w", err) } names = append(names, name) } @@ -1583,7 +1579,7 @@ func (r *Reader) LabelValueFor(ctx context.Context, id storage.SeriesRef, label d := encoding.NewDecbufUvarintAt(r.b, int(offset), castagnoliTable) buf := d.Get() if d.Err() != nil { - return "", errors.Wrap(d.Err(), "label values for") + return "", fmt.Errorf("label values for: %w", d.Err()) } value, err := r.dec.LabelValueFor(ctx, buf, label) @@ -1610,7 +1606,11 @@ func (r *Reader) Series(id storage.SeriesRef, builder *labels.ScratchBuilder, ch if d.Err() != nil { return d.Err() } - return errors.Wrap(r.dec.Series(d.Get(), builder, chks), "read series") + err := r.dec.Series(d.Get(), builder, chks) + if err != nil { + return fmt.Errorf("read series: %w", err) + } + return nil } func (r *Reader) Postings(ctx context.Context, name string, values ...string) (Postings, error) { @@ -1629,7 +1629,7 @@ func (r *Reader) Postings(ctx context.Context, name string, values ...string) (P d := encoding.NewDecbufAt(r.b, int(postingsOff), castagnoliTable) _, p, err := r.dec.Postings(d.Get()) if err != nil { - return nil, errors.Wrap(err, "decode postings") + return nil, fmt.Errorf("decode postings: %w", err) } res = append(res, p) } @@ -1691,7 +1691,7 @@ func (r *Reader) Postings(ctx context.Context, name string, values ...string) (P d2 := encoding.NewDecbufAt(r.b, int(postingsOff), castagnoliTable) _, p, err := r.dec.Postings(d2.Get()) if err != nil { - return nil, errors.Wrap(err, "decode postings") + return nil, fmt.Errorf("decode postings: %w", err) } res = append(res, p) } @@ -1707,10 +1707,10 @@ func (r *Reader) Postings(ctx context.Context, name string, values ...string) (P } } if d.Err() != nil { - return nil, errors.Wrap(d.Err(), "get postings offset entry") + return nil, fmt.Errorf("get postings offset entry: %w", d.Err()) } if ctx.Err() != nil { - return nil, errors.Wrap(ctx.Err(), "get postings offset entry") + return nil, fmt.Errorf("get postings offset entry: %w", ctx.Err()) } } @@ -1729,10 +1729,10 @@ func (r *Reader) Size() int64 { } // LabelNames returns all the unique label names present in the index. -// TODO(twilkie) implement support for matchers +// TODO(twilkie) implement support for matchers. func (r *Reader) LabelNames(_ context.Context, matchers ...*labels.Matcher) ([]string, error) { if len(matchers) > 0 { - return nil, errors.Errorf("matchers parameter is not implemented: %+v", matchers) + return nil, fmt.Errorf("matchers parameter is not implemented: %+v", matchers) } labelNames := make([]string, 0, len(r.postings)) @@ -1803,7 +1803,7 @@ func (dec *Decoder) LabelNamesOffsetsFor(b []byte) ([]uint32, error) { _ = d.Uvarint() // skip the label value if d.Err() != nil { - return nil, errors.Wrap(d.Err(), "read series label offsets") + return nil, fmt.Errorf("read series label offsets: %w", d.Err()) } } @@ -1820,18 +1820,18 @@ func (dec *Decoder) LabelValueFor(ctx context.Context, b []byte, label string) ( lvo := uint32(d.Uvarint()) if d.Err() != nil { - return "", errors.Wrap(d.Err(), "read series label offsets") + return "", fmt.Errorf("read series label offsets: %w", d.Err()) } ln, err := dec.LookupSymbol(ctx, lno) if err != nil { - return "", errors.Wrap(err, "lookup label name") + return "", fmt.Errorf("lookup label name: %w", err) } if ln == label { lv, err := dec.LookupSymbol(ctx, lvo) if err != nil { - return "", errors.Wrap(err, "lookup label value") + return "", fmt.Errorf("lookup label value: %w", err) } return lv, nil @@ -1856,16 +1856,16 @@ func (dec *Decoder) Series(b []byte, builder *labels.ScratchBuilder, chks *[]chu lvo := uint32(d.Uvarint()) if d.Err() != nil { - return errors.Wrap(d.Err(), "read series label offsets") + return fmt.Errorf("read series label offsets: %w", d.Err()) } ln, err := dec.LookupSymbol(context.TODO(), lno) if err != nil { - return errors.Wrap(err, "lookup label name") + return fmt.Errorf("lookup label name: %w", err) } lv, err := dec.LookupSymbol(context.TODO(), lvo) if err != nil { - return errors.Wrap(err, "lookup label value") + return fmt.Errorf("lookup label value: %w", err) } builder.Add(ln, lv) @@ -1897,7 +1897,7 @@ func (dec *Decoder) Series(b []byte, builder *labels.ScratchBuilder, chks *[]chu t0 = maxt if d.Err() != nil { - return errors.Wrapf(d.Err(), "read meta for chunk %d", i) + return fmt.Errorf("read meta for chunk %d: %w", i, d.Err()) } *chks = append(*chks, chunks.Meta{ diff --git a/vendor/github.com/prometheus/prometheus/tsdb/index/postings.go b/vendor/github.com/prometheus/prometheus/tsdb/index/postings.go index f79a8d4cfc..c839574276 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/index/postings.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/index/postings.go @@ -17,12 +17,12 @@ import ( "container/heap" "context" "encoding/binary" + "fmt" "runtime" "sort" "strings" "sync" - "github.com/pkg/errors" "golang.org/x/exp/slices" "github.com/prometheus/prometheus/model/labels" @@ -927,7 +927,7 @@ func (h *postingsWithIndexHeap) next() error { } if err := pi.p.Err(); err != nil { - return errors.Wrapf(err, "postings %d", pi.index) + return fmt.Errorf("postings %d: %w", pi.index, err) } h.popIndex() return nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/index/postingsstats.go b/vendor/github.com/prometheus/prometheus/tsdb/index/postingsstats.go index 70622b3d27..0ad4e08575 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/index/postingsstats.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/index/postingsstats.go @@ -63,6 +63,15 @@ func (m *maxHeap) push(item Stat) { } func (m *maxHeap) get() []Stat { - slices.SortFunc(m.Items, func(a, b Stat) int { return int(b.Count - a.Count) }) + slices.SortFunc(m.Items, func(a, b Stat) int { + switch { + case b.Count < a.Count: + return -1 + case b.Count > a.Count: + return 1 + default: + return 0 + } + }) return m.Items } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head.go b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head.go index 45827889e6..7f2110fa65 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head.go @@ -17,7 +17,10 @@ import ( "fmt" "sort" + "github.com/oklog/ulid" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/tombstones" ) @@ -111,22 +114,27 @@ type OOORangeHead struct { // the timerange of the query and having preexisting pointers to the first // and last timestamp help with that. mint, maxt int64 + + isoState *oooIsolationState } -func NewOOORangeHead(head *Head, mint, maxt int64) *OOORangeHead { +func NewOOORangeHead(head *Head, mint, maxt int64, minRef chunks.ChunkDiskMapperRef) *OOORangeHead { + isoState := head.oooIso.TrackReadAfter(minRef) + return &OOORangeHead{ - head: head, - mint: mint, - maxt: maxt, + head: head, + mint: mint, + maxt: maxt, + isoState: isoState, } } func (oh *OOORangeHead) Index() (IndexReader, error) { - return NewOOOHeadIndexReader(oh.head, oh.mint, oh.maxt), nil + return NewOOOHeadIndexReader(oh.head, oh.mint, oh.maxt, oh.isoState.minRef), nil } func (oh *OOORangeHead) Chunks() (ChunkReader, error) { - return NewOOOHeadChunkReader(oh.head, oh.mint, oh.maxt), nil + return NewOOOHeadChunkReader(oh.head, oh.mint, oh.maxt, oh.isoState), nil } func (oh *OOORangeHead) Tombstones() (tombstones.Reader, error) { @@ -135,13 +143,13 @@ func (oh *OOORangeHead) Tombstones() (tombstones.Reader, error) { return tombstones.NewMemTombstones(), nil } +var oooRangeHeadULID = ulid.MustParse("0000000000XXXX000RANGEHEAD") + func (oh *OOORangeHead) Meta() BlockMeta { - var id [16]byte - copy(id[:], "____ooo_head____") return BlockMeta{ MinTime: oh.mint, MaxTime: oh.maxt, - ULID: id, + ULID: oooRangeHeadULID, Stats: BlockStats{ NumSeries: oh.head.NumSeries(), }, diff --git a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go index 198bc4f2f4..440130f7db 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go @@ -11,7 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive // Many unused function arguments in this file by design. package tsdb import ( @@ -19,6 +18,7 @@ import ( "errors" "math" + "github.com/oklog/ulid" "golang.org/x/exp/slices" "github.com/prometheus/prometheus/model/labels" @@ -38,26 +38,29 @@ var _ IndexReader = &OOOHeadIndexReader{} // decided to do this to avoid code duplication. // The only methods that change are the ones about getting Series and Postings. type OOOHeadIndexReader struct { - *headIndexReader // A reference to the headIndexReader so we can reuse as many interface implementation as possible. + *headIndexReader // A reference to the headIndexReader so we can reuse as many interface implementation as possible. + lastGarbageCollectedMmapRef chunks.ChunkDiskMapperRef } -func NewOOOHeadIndexReader(head *Head, mint, maxt int64) *OOOHeadIndexReader { +func NewOOOHeadIndexReader(head *Head, mint, maxt int64, lastGarbageCollectedMmapRef chunks.ChunkDiskMapperRef) *OOOHeadIndexReader { hr := &headIndexReader{ head: head, mint: mint, maxt: maxt, } - return &OOOHeadIndexReader{hr} + return &OOOHeadIndexReader{hr, lastGarbageCollectedMmapRef} } func (oh *OOOHeadIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error { - return oh.series(ref, builder, chks, 0) + return oh.series(ref, builder, chks, oh.lastGarbageCollectedMmapRef, 0) } -// The passed lastMmapRef tells upto what max m-map chunk that we can consider. -// If it is 0, it means all chunks need to be considered. -// If it is non-0, then the oooHeadChunk must not be considered. -func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta, lastMmapRef chunks.ChunkDiskMapperRef) error { +// lastGarbageCollectedMmapRef gives the last mmap chunk that may be being garbage collected and so +// any chunk at or before this ref will not be considered. 0 disables this check. +// +// maxMmapRef tells upto what max m-map chunk that we can consider. If it is non-0, then +// the oooHeadChunk will not be considered. +func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta, lastGarbageCollectedMmapRef, maxMmapRef chunks.ChunkDiskMapperRef) error { s := oh.head.series.getByID(chunks.HeadSeriesRef(ref)) if s == nil { @@ -112,14 +115,14 @@ func (oh *OOOHeadIndexReader) series(ref storage.SeriesRef, builder *labels.Scra // so we can set the correct markers. if s.ooo.oooHeadChunk != nil { c := s.ooo.oooHeadChunk - if c.OverlapsClosedInterval(oh.mint, oh.maxt) && lastMmapRef == 0 { + if c.OverlapsClosedInterval(oh.mint, oh.maxt) && maxMmapRef == 0 { ref := chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(len(s.ooo.oooMmappedChunks)))) addChunk(c.minTime, c.maxTime, ref) } } for i := len(s.ooo.oooMmappedChunks) - 1; i >= 0; i-- { c := s.ooo.oooMmappedChunks[i] - if c.OverlapsClosedInterval(oh.mint, oh.maxt) && (lastMmapRef == 0 || lastMmapRef.GreaterThanOrEqualTo(c.ref)) { + if c.OverlapsClosedInterval(oh.mint, oh.maxt) && (maxMmapRef == 0 || maxMmapRef.GreaterThanOrEqualTo(c.ref)) && (lastGarbageCollectedMmapRef == 0 || c.ref.GreaterThan(lastGarbageCollectedMmapRef)) { ref := chunks.ChunkRef(chunks.NewHeadChunkRef(s.ref, s.oooHeadChunkID(i))) addChunk(c.minTime, c.maxTime, ref) } @@ -178,17 +181,39 @@ type chunkMetaAndChunkDiskMapperRef struct { } func refLessByMinTimeAndMinRef(a, b chunkMetaAndChunkDiskMapperRef) int { - if a.meta.MinTime == b.meta.MinTime { - return int(a.meta.Ref - b.meta.Ref) + switch { + case a.meta.MinTime < b.meta.MinTime: + return -1 + case a.meta.MinTime > b.meta.MinTime: + return 1 + } + + switch { + case a.meta.Ref < b.meta.Ref: + return -1 + case a.meta.Ref > b.meta.Ref: + return 1 + default: + return 0 } - return int(a.meta.MinTime - b.meta.MinTime) } func lessByMinTimeAndMinRef(a, b chunks.Meta) int { - if a.MinTime == b.MinTime { - return int(a.Ref - b.Ref) + switch { + case a.MinTime < b.MinTime: + return -1 + case a.MinTime > b.MinTime: + return 1 + } + + switch { + case a.Ref < b.Ref: + return -1 + case a.Ref > b.Ref: + return 1 + default: + return 0 } - return int(a.MinTime - b.MinTime) } func (oh *OOOHeadIndexReader) Postings(ctx context.Context, name string, values ...string) (index.Postings, error) { @@ -210,46 +235,51 @@ func (oh *OOOHeadIndexReader) Postings(ctx context.Context, name string, values type OOOHeadChunkReader struct { head *Head mint, maxt int64 + isoState *oooIsolationState } -func NewOOOHeadChunkReader(head *Head, mint, maxt int64) *OOOHeadChunkReader { +func NewOOOHeadChunkReader(head *Head, mint, maxt int64, isoState *oooIsolationState) *OOOHeadChunkReader { return &OOOHeadChunkReader{ - head: head, - mint: mint, - maxt: maxt, + head: head, + mint: mint, + maxt: maxt, + isoState: isoState, } } -func (cr OOOHeadChunkReader) Chunk(meta chunks.Meta) (chunkenc.Chunk, error) { +func (cr OOOHeadChunkReader) ChunkOrIterable(meta chunks.Meta) (chunkenc.Chunk, chunkenc.Iterable, error) { sid, _ := chunks.HeadChunkRef(meta.Ref).Unpack() s := cr.head.series.getByID(sid) // This means that the series has been garbage collected. if s == nil { - return nil, storage.ErrNotFound + return nil, nil, storage.ErrNotFound } s.Lock() if s.ooo == nil { // There is no OOO data for this series. s.Unlock() - return nil, storage.ErrNotFound + return nil, nil, storage.ErrNotFound } - c, err := s.oooMergedChunk(meta, cr.head.chunkDiskMapper, cr.mint, cr.maxt) + mc, err := s.oooMergedChunks(meta, cr.head.chunkDiskMapper, cr.mint, cr.maxt) s.Unlock() if err != nil { - return nil, err + return nil, nil, err } // This means that the query range did not overlap with the requested chunk. - if len(c.chunks) == 0 { - return nil, storage.ErrNotFound + if len(mc.chunkIterables) == 0 { + return nil, nil, storage.ErrNotFound } - return c, nil + return nil, mc, nil } func (cr OOOHeadChunkReader) Close() error { + if cr.isoState != nil { + cr.isoState.Close() + } return nil } @@ -284,7 +314,7 @@ func NewOOOCompactionHead(ctx context.Context, head *Head) (*OOOCompactionHead, ch.lastWBLFile = lastWBLFile } - ch.oooIR = NewOOOHeadIndexReader(head, math.MinInt64, math.MaxInt64) + ch.oooIR = NewOOOHeadIndexReader(head, math.MinInt64, math.MaxInt64, 0) n, v := index.AllPostingsKey() // TODO: verify this gets only ooo samples. @@ -343,20 +373,20 @@ func (ch *OOOCompactionHead) Index() (IndexReader, error) { } func (ch *OOOCompactionHead) Chunks() (ChunkReader, error) { - return NewOOOHeadChunkReader(ch.oooIR.head, ch.oooIR.mint, ch.oooIR.maxt), nil + return NewOOOHeadChunkReader(ch.oooIR.head, ch.oooIR.mint, ch.oooIR.maxt, nil), nil } func (ch *OOOCompactionHead) Tombstones() (tombstones.Reader, error) { return tombstones.NewMemTombstones(), nil } +var oooCompactionHeadULID = ulid.MustParse("0000000000XX000COMPACTHEAD") + func (ch *OOOCompactionHead) Meta() BlockMeta { - var id [16]byte - copy(id[:], "copy(id[:], \"ooo_compact_head\")") return BlockMeta{ MinTime: ch.mint, MaxTime: ch.maxt, - ULID: id, + ULID: oooCompactionHeadULID, Stats: BlockStats{ NumSeries: uint64(len(ch.postings)), }, @@ -369,7 +399,7 @@ func (ch *OOOCompactionHead) Meta() BlockMeta { // Only the method of BlockReader interface are valid for the cloned OOOCompactionHead. func (ch *OOOCompactionHead) CloneForTimeRange(mint, maxt int64) *OOOCompactionHead { return &OOOCompactionHead{ - oooIR: NewOOOHeadIndexReader(ch.oooIR.head, mint, maxt), + oooIR: NewOOOHeadIndexReader(ch.oooIR.head, mint, maxt, 0), lastMmapRef: ch.lastMmapRef, postings: ch.postings, chunkRange: ch.chunkRange, @@ -411,7 +441,7 @@ func (ir *OOOCompactionHeadIndexReader) SortedPostings(p index.Postings) index.P } func (ir *OOOCompactionHeadIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error { - return ir.ch.oooIR.series(ref, builder, chks, ir.ch.lastMmapRef) + return ir.ch.oooIR.series(ref, builder, chks, 0, ir.ch.lastMmapRef) } func (ir *OOOCompactionHeadIndexReader) SortedLabelValues(_ context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/ooo_isolation.go b/vendor/github.com/prometheus/prometheus/tsdb/ooo_isolation.go new file mode 100644 index 0000000000..3e3e165a0a --- /dev/null +++ b/vendor/github.com/prometheus/prometheus/tsdb/ooo_isolation.go @@ -0,0 +1,79 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tsdb + +import ( + "container/list" + "sync" + + "github.com/prometheus/prometheus/tsdb/chunks" +) + +type oooIsolation struct { + mtx sync.RWMutex + openReads *list.List +} + +type oooIsolationState struct { + i *oooIsolation + e *list.Element + + minRef chunks.ChunkDiskMapperRef +} + +func newOOOIsolation() *oooIsolation { + return &oooIsolation{ + openReads: list.New(), + } +} + +// HasOpenReadsAtOrBefore returns true if this oooIsolation is aware of any reads that use +// chunks with reference at or before ref. +func (i *oooIsolation) HasOpenReadsAtOrBefore(ref chunks.ChunkDiskMapperRef) bool { + i.mtx.RLock() + defer i.mtx.RUnlock() + + for e := i.openReads.Front(); e != nil; e = e.Next() { + s := e.Value.(*oooIsolationState) + + if ref.GreaterThan(s.minRef) { + return true + } + } + + return false +} + +// TrackReadAfter records a read that uses chunks with reference after minRef. +// +// The caller must ensure that the returned oooIsolationState is eventually closed when +// the read is complete. +func (i *oooIsolation) TrackReadAfter(minRef chunks.ChunkDiskMapperRef) *oooIsolationState { + s := &oooIsolationState{ + i: i, + minRef: minRef, + } + + i.mtx.Lock() + s.e = i.openReads.PushBack(s) + i.mtx.Unlock() + + return s +} + +func (s oooIsolationState) Close() { + s.i.mtx.Lock() + s.i.openReads.Remove(s.e) + s.i.mtx.Unlock() +} diff --git a/vendor/github.com/prometheus/prometheus/tsdb/querier.go b/vendor/github.com/prometheus/prometheus/tsdb/querier.go index a832c0d1ba..6584d7da0a 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/querier.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/querier.go @@ -632,36 +632,42 @@ func (b *blockBaseSeriesSet) Warnings() annotations.Annotations { return nil } // populateWithDelGenericSeriesIterator assumes that chunks that would be fully // removed by intervals are filtered out in previous phase. // -// On each iteration currChkMeta is available. If currDelIter is not nil, it -// means that the chunk iterator in currChkMeta is invalid and a chunk rewrite -// is needed, for which currDelIter should be used. +// On each iteration currMeta is available. If currDelIter is not nil, it +// means that the chunk in currMeta is invalid and a chunk rewrite is needed, +// for which currDelIter should be used. type populateWithDelGenericSeriesIterator struct { blockID ulid.ULID - chunks ChunkReader - // chks are expected to be sorted by minTime and should be related to + cr ChunkReader + // metas are expected to be sorted by minTime and should be related to // the same, single series. - chks []chunks.Meta + // It's possible for a single chunks.Meta to refer to multiple chunks. + // cr.ChunkOrIterator() would return an iterable and a nil chunk in this + // case. + metas []chunks.Meta - i int // Index into chks; -1 if not started yet. + i int // Index into metas; -1 if not started yet. err error bufIter DeletedIterator // Retained for memory re-use. currDelIter may point here. intervals tombstones.Intervals currDelIter chunkenc.Iterator - currChkMeta chunks.Meta + // currMeta is the current chunks.Meta from metas. currMeta.Chunk is set to + // the chunk returned from cr.ChunkOrIterable(). As that can return a nil + // chunk, currMeta.Chunk is not always guaranteed to be set. + currMeta chunks.Meta } func (p *populateWithDelGenericSeriesIterator) reset(blockID ulid.ULID, cr ChunkReader, chks []chunks.Meta, intervals tombstones.Intervals) { p.blockID = blockID - p.chunks = cr - p.chks = chks + p.cr = cr + p.metas = chks p.i = -1 p.err = nil // Note we don't touch p.bufIter.Iter; it is holding on to an iterator we might reuse in next(). p.bufIter.Intervals = p.bufIter.Intervals[:0] p.intervals = intervals p.currDelIter = nil - p.currChkMeta = chunks.Meta{} + p.currMeta = chunks.Meta{} } // If copyHeadChunk is true, then the head chunk (i.e. the in-memory chunk of the TSDB) @@ -669,43 +675,54 @@ func (p *populateWithDelGenericSeriesIterator) reset(blockID ulid.ULID, cr Chunk // However, if the deletion intervals overlaps with the head chunk, then the head chunk is // not copied irrespective of copyHeadChunk because it will be re-encoded later anyway. func (p *populateWithDelGenericSeriesIterator) next(copyHeadChunk bool) bool { - if p.err != nil || p.i >= len(p.chks)-1 { + if p.err != nil || p.i >= len(p.metas)-1 { return false } p.i++ - p.currChkMeta = p.chks[p.i] + p.currMeta = p.metas[p.i] p.bufIter.Intervals = p.bufIter.Intervals[:0] for _, interval := range p.intervals { - if p.currChkMeta.OverlapsClosedInterval(interval.Mint, interval.Maxt) { + if p.currMeta.OverlapsClosedInterval(interval.Mint, interval.Maxt) { p.bufIter.Intervals = p.bufIter.Intervals.Add(interval) } } - hcr, ok := p.chunks.(*headChunkReader) + hcr, ok := p.cr.(*headChunkReader) + var iterable chunkenc.Iterable if ok && copyHeadChunk && len(p.bufIter.Intervals) == 0 { // ChunkWithCopy will copy the head chunk. var maxt int64 - p.currChkMeta.Chunk, maxt, p.err = hcr.ChunkWithCopy(p.currChkMeta) + p.currMeta.Chunk, maxt, p.err = hcr.ChunkWithCopy(p.currMeta) // For the in-memory head chunk the index reader sets maxt as MaxInt64. We fix it here. - p.currChkMeta.MaxTime = maxt + p.currMeta.MaxTime = maxt } else { - p.currChkMeta.Chunk, p.err = p.chunks.Chunk(p.currChkMeta) + p.currMeta.Chunk, iterable, p.err = p.cr.ChunkOrIterable(p.currMeta) } + if p.err != nil { - p.err = errors.Wrapf(p.err, "cannot populate chunk %d from block %s", p.currChkMeta.Ref, p.blockID.String()) + p.err = errors.Wrapf(p.err, "cannot populate chunk %d from block %s", p.currMeta.Ref, p.blockID.String()) return false } - if len(p.bufIter.Intervals) == 0 { - // If there is no overlap with deletion intervals, we can take chunk as it is. - p.currDelIter = nil + // Use the single chunk if possible. + if p.currMeta.Chunk != nil { + if len(p.bufIter.Intervals) == 0 { + // If there is no overlap with deletion intervals and a single chunk is + // returned, we can take chunk as it is. + p.currDelIter = nil + return true + } + // Otherwise we need to iterate over the samples in the single chunk + // and create new chunks. + p.bufIter.Iter = p.currMeta.Chunk.Iterator(p.bufIter.Iter) + p.currDelIter = &p.bufIter return true } - // We don't want the full chunk, take just a part of it. - p.bufIter.Iter = p.currChkMeta.Chunk.Iterator(p.bufIter.Iter) + // Otherwise, use the iterable to create an iterator. + p.bufIter.Iter = iterable.Iterator(p.bufIter.Iter) p.currDelIter = &p.bufIter return true } @@ -765,7 +782,7 @@ func (p *populateWithDelSeriesIterator) Next() chunkenc.ValueType { if p.currDelIter != nil { p.curr = p.currDelIter } else { - p.curr = p.currChkMeta.Chunk.Iterator(p.curr) + p.curr = p.currMeta.Chunk.Iterator(p.curr) } if valueType := p.curr.Next(); valueType != chunkenc.ValNone { return valueType @@ -817,22 +834,69 @@ func (p *populateWithDelSeriesIterator) Err() error { type populateWithDelChunkSeriesIterator struct { populateWithDelGenericSeriesIterator - curr chunks.Meta + // currMetaWithChunk is current meta with its chunk field set. This meta + // is guaranteed to map to a single chunk. This differs from + // populateWithDelGenericSeriesIterator.currMeta as that + // could refer to multiple chunks. + currMetaWithChunk chunks.Meta + + // chunksFromIterable stores the chunks created from iterating through + // the iterable returned by cr.ChunkOrIterable() (with deleted samples + // removed). + chunksFromIterable []chunks.Meta + chunksFromIterableIdx int } func (p *populateWithDelChunkSeriesIterator) reset(blockID ulid.ULID, cr ChunkReader, chks []chunks.Meta, intervals tombstones.Intervals) { p.populateWithDelGenericSeriesIterator.reset(blockID, cr, chks, intervals) - p.curr = chunks.Meta{} + p.currMetaWithChunk = chunks.Meta{} + p.chunksFromIterable = p.chunksFromIterable[:0] + p.chunksFromIterableIdx = -1 } func (p *populateWithDelChunkSeriesIterator) Next() bool { - if !p.next(true) { - return false + if p.currMeta.Chunk == nil { + // If we've been creating chunks from the iterable, check if there are + // any more chunks to iterate through. + if p.chunksFromIterableIdx < len(p.chunksFromIterable)-1 { + p.chunksFromIterableIdx++ + p.currMetaWithChunk = p.chunksFromIterable[p.chunksFromIterableIdx] + return true + } } - p.curr = p.currChkMeta - if p.currDelIter == nil { - return true + + // Move to the next chunk/deletion iterator. + // This is a for loop as if the current p.currDelIter returns no samples + // (which means a chunk won't be created), there still might be more + // samples/chunks from the rest of p.metas. + for p.next(true) { + if p.currDelIter == nil { + p.currMetaWithChunk = p.currMeta + return true + } + + if p.currMeta.Chunk != nil { + // If ChunkOrIterable() returned a non-nil chunk, the samples in + // p.currDelIter will only form one chunk, as the only change + // p.currDelIter might make is deleting some samples. + if p.populateCurrForSingleChunk() { + return true + } + } else { + // If ChunkOrIterable() returned an iterable, multiple chunks may be + // created from the samples in p.currDelIter. + if p.populateChunksFromIterable() { + return true + } + } + } + return false +} + +// populateCurrForSingleChunk sets the fields within p.currMetaWithChunk. This +// should be called if the samples in p.currDelIter only form one chunk. +func (p *populateWithDelChunkSeriesIterator) populateCurrForSingleChunk() bool { valueType := p.currDelIter.Next() if valueType == chunkenc.ValNone { if err := p.currDelIter.Err(); err != nil { @@ -840,9 +904,9 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { } return false } - p.curr.MinTime = p.currDelIter.AtT() + p.currMetaWithChunk.MinTime = p.currDelIter.AtT() - // Re-encode the chunk if iterator is provider. This means that it has + // Re-encode the chunk if iterator is provided. This means that it has // some samples to be deleted or chunk is opened. var ( newChunk chunkenc.Chunk @@ -900,7 +964,7 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { } } default: - err = fmt.Errorf("populateWithDelChunkSeriesIterator: value type %v unsupported", valueType) + err = fmt.Errorf("populateCurrForSingleChunk: value type %v unsupported", valueType) } if err != nil { @@ -912,12 +976,127 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool { return false } - p.curr.Chunk = newChunk - p.curr.MaxTime = t + p.currMetaWithChunk.Chunk = newChunk + p.currMetaWithChunk.MaxTime = t + return true +} + +// populateChunksFromIterable reads the samples from currDelIter to create +// chunks for chunksFromIterable. It also sets p.currMetaWithChunk to the first +// chunk. +func (p *populateWithDelChunkSeriesIterator) populateChunksFromIterable() bool { + p.chunksFromIterable = p.chunksFromIterable[:0] + p.chunksFromIterableIdx = -1 + + firstValueType := p.currDelIter.Next() + if firstValueType == chunkenc.ValNone { + if err := p.currDelIter.Err(); err != nil { + p.err = errors.Wrap(err, "populateChunksFromIterable: no samples could be read") + return false + } + return false + } + + var ( + // t is the timestamp for the current sample. + t int64 + cmint int64 + cmaxt int64 + + currentChunk chunkenc.Chunk + + app chunkenc.Appender + + newChunk chunkenc.Chunk + recoded bool + + err error + ) + + prevValueType := chunkenc.ValNone + + for currentValueType := firstValueType; currentValueType != chunkenc.ValNone; currentValueType = p.currDelIter.Next() { + // Check if the encoding has changed (i.e. we need to create a new + // chunk as chunks can't have multiple encoding types). + // For the first sample, the following condition will always be true as + // ValNoneNone != ValFloat | ValHistogram | ValFloatHistogram. + if currentValueType != prevValueType { + if prevValueType != chunkenc.ValNone { + p.chunksFromIterable = append(p.chunksFromIterable, chunks.Meta{Chunk: currentChunk, MinTime: cmint, MaxTime: cmaxt}) + } + cmint = p.currDelIter.AtT() + if currentChunk, err = currentValueType.NewChunk(); err != nil { + break + } + if app, err = currentChunk.Appender(); err != nil { + break + } + } + + switch currentValueType { + case chunkenc.ValFloat: + { + var v float64 + t, v = p.currDelIter.At() + app.Append(t, v) + } + case chunkenc.ValHistogram: + { + var v *histogram.Histogram + t, v = p.currDelIter.AtHistogram() + // No need to set prevApp as AppendHistogram will set the + // counter reset header for the appender that's returned. + newChunk, recoded, app, err = app.AppendHistogram(nil, t, v, false) + } + case chunkenc.ValFloatHistogram: + { + var v *histogram.FloatHistogram + t, v = p.currDelIter.AtFloatHistogram() + // No need to set prevApp as AppendHistogram will set the + // counter reset header for the appender that's returned. + newChunk, recoded, app, err = app.AppendFloatHistogram(nil, t, v, false) + } + } + + if err != nil { + break + } + + if newChunk != nil { + if !recoded { + p.chunksFromIterable = append(p.chunksFromIterable, chunks.Meta{Chunk: currentChunk, MinTime: cmint, MaxTime: cmaxt}) + } + currentChunk = newChunk + cmint = t + } + + cmaxt = t + prevValueType = currentValueType + } + + if err != nil { + p.err = errors.Wrap(err, "populateChunksFromIterable: error when writing new chunks") + return false + } + if err = p.currDelIter.Err(); err != nil { + p.err = errors.Wrap(err, "populateChunksFromIterable: currDelIter error when writing new chunks") + return false + } + + if prevValueType != chunkenc.ValNone { + p.chunksFromIterable = append(p.chunksFromIterable, chunks.Meta{Chunk: currentChunk, MinTime: cmint, MaxTime: cmaxt}) + } + + if len(p.chunksFromIterable) == 0 { + return false + } + + p.currMetaWithChunk = p.chunksFromIterable[0] + p.chunksFromIterableIdx = 0 return true } -func (p *populateWithDelChunkSeriesIterator) At() chunks.Meta { return p.curr } +func (p *populateWithDelChunkSeriesIterator) At() chunks.Meta { return p.currMetaWithChunk } // blockSeriesSet allows to iterate over sorted, populated series with applied tombstones. // Series with all deleted chunks are still present as Series with no samples. @@ -1117,8 +1296,8 @@ func newNopChunkReader() ChunkReader { } } -func (cr nopChunkReader) Chunk(chunks.Meta) (chunkenc.Chunk, error) { - return cr.emptyChunk, nil +func (cr nopChunkReader) ChunkOrIterable(chunks.Meta) (chunkenc.Chunk, chunkenc.Iterable, error) { + return cr.emptyChunk, nil, nil } func (cr nopChunkReader) Close() error { return nil } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/record/record.go b/vendor/github.com/prometheus/prometheus/tsdb/record/record.go index 4cd51d46c0..42a656dfe8 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/record/record.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/record/record.go @@ -16,10 +16,10 @@ package record import ( + "errors" + "fmt" "math" - "github.com/pkg/errors" - "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/textparse" @@ -229,7 +229,7 @@ func (d *Decoder) Series(rec []byte, series []RefSeries) ([]RefSeries, error) { return nil, dec.Err() } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return series, nil } @@ -272,20 +272,19 @@ func (d *Decoder) Metadata(rec []byte, metadata []RefMetadata) ([]RefMetadata, e return nil, dec.Err() } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return metadata, nil } // DecodeLabels decodes one set of labels from buf. func (d *Decoder) DecodeLabels(dec *encoding.Decbuf) labels.Labels { - // TODO: reconsider if this function could be pushed down into labels.Labels to be more efficient. d.builder.Reset() nLabels := dec.Uvarint() for i := 0; i < nLabels; i++ { - lName := dec.UvarintStr() - lValue := dec.UvarintStr() - d.builder.Add(lName, lValue) + lName := dec.UvarintBytes() + lValue := dec.UvarintBytes() + d.builder.UnsafeAddBytes(lName, lValue) } return d.builder.Labels() } @@ -304,6 +303,10 @@ func (d *Decoder) Samples(rec []byte, samples []RefSample) ([]RefSample, error) baseRef = dec.Be64() baseTime = dec.Be64int64() ) + // Allow 1 byte for each varint and 8 for the value; the output slice must be at least that big. + if minSize := dec.Len() / (1 + 1 + 8); cap(samples) < minSize { + samples = make([]RefSample, 0, minSize) + } for len(dec.B) > 0 && dec.Err() == nil { dref := dec.Varint64() dtime := dec.Varint64() @@ -317,10 +320,10 @@ func (d *Decoder) Samples(rec []byte, samples []RefSample) ([]RefSample, error) } if dec.Err() != nil { - return nil, errors.Wrapf(dec.Err(), "decode error after %d samples", len(samples)) + return nil, fmt.Errorf("decode error after %d samples: %w", len(samples), dec.Err()) } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return samples, nil } @@ -344,7 +347,7 @@ func (d *Decoder) Tombstones(rec []byte, tstones []tombstones.Stone) ([]tombston return nil, dec.Err() } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return tstones, nil } @@ -382,10 +385,10 @@ func (d *Decoder) ExemplarsFromBuffer(dec *encoding.Decbuf, exemplars []RefExemp } if dec.Err() != nil { - return nil, errors.Wrapf(dec.Err(), "decode error after %d exemplars", len(exemplars)) + return nil, fmt.Errorf("decode error after %d exemplars: %w", len(exemplars), dec.Err()) } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return exemplars, nil } @@ -410,10 +413,10 @@ func (d *Decoder) MmapMarkers(rec []byte, markers []RefMmapMarker) ([]RefMmapMar } if dec.Err() != nil { - return nil, errors.Wrapf(dec.Err(), "decode error after %d mmap markers", len(markers)) + return nil, fmt.Errorf("decode error after %d mmap markers: %w", len(markers), dec.Err()) } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return markers, nil } @@ -446,10 +449,10 @@ func (d *Decoder) HistogramSamples(rec []byte, histograms []RefHistogramSample) } if dec.Err() != nil { - return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(histograms)) + return nil, fmt.Errorf("decode error after %d histograms: %w", len(histograms), dec.Err()) } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return histograms, nil } @@ -528,10 +531,10 @@ func (d *Decoder) FloatHistogramSamples(rec []byte, histograms []RefFloatHistogr } if dec.Err() != nil { - return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(histograms)) + return nil, fmt.Errorf("decode error after %d histograms: %w", len(histograms), dec.Err()) } if len(dec.B) > 0 { - return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return nil, fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return histograms, nil } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/repair.go b/vendor/github.com/prometheus/prometheus/tsdb/repair.go index 0c2e08791c..0811164541 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/repair.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/repair.go @@ -15,6 +15,7 @@ package tsdb import ( "encoding/json" + "fmt" "io" "os" "path/filepath" @@ -124,7 +125,7 @@ func readBogusMetaFile(dir string) (*BlockMeta, error) { return nil, err } if m.Version != metaVersion1 && m.Version != 2 { - return nil, errors.Errorf("unexpected meta file version %d", m.Version) + return nil, fmt.Errorf("unexpected meta file version %d", m.Version) } return &m, nil } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go b/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go index 94daf51953..4cea5005db 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/tombstones/tombstones.go @@ -15,6 +15,7 @@ package tombstones import ( "encoding/binary" + "errors" "fmt" "hash" "hash/crc32" @@ -26,7 +27,6 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/pkg/errors" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/encoding" @@ -109,17 +109,17 @@ func WriteFile(logger log.Logger, dir string, tr Reader) (int64, error) { bytes, err := Encode(tr) if err != nil { - return 0, errors.Wrap(err, "encoding tombstones") + return 0, fmt.Errorf("encoding tombstones: %w", err) } // Ignore first byte which is the format type. We do this for compatibility. if _, err := hash.Write(bytes[tombstoneFormatVersionSize:]); err != nil { - return 0, errors.Wrap(err, "calculating hash for tombstones") + return 0, fmt.Errorf("calculating hash for tombstones: %w", err) } n, err = f.Write(bytes) if err != nil { - return 0, errors.Wrap(err, "writing tombstones") + return 0, fmt.Errorf("writing tombstones: %w", err) } size += n @@ -161,7 +161,7 @@ func Encode(tr Reader) ([]byte, error) { func Decode(b []byte) (Reader, error) { d := &encoding.Decbuf{B: b} if flag := d.Byte(); flag != tombstoneFormatV1 { - return nil, errors.Errorf("invalid tombstone format %x", flag) + return nil, fmt.Errorf("invalid tombstone format %x", flag) } if d.Err() != nil { @@ -199,7 +199,7 @@ func ReadTombstones(dir string) (Reader, int64, error) { } if len(b) < tombstonesHeaderSize { - return nil, 0, errors.Wrap(encoding.ErrInvalidSize, "tombstones header") + return nil, 0, fmt.Errorf("tombstones header: %w", encoding.ErrInvalidSize) } d := &encoding.Decbuf{B: b[:len(b)-tombstonesCRCSize]} @@ -211,7 +211,7 @@ func ReadTombstones(dir string) (Reader, int64, error) { hash := newCRC32() // Ignore first byte which is the format type. if _, err := hash.Write(d.Get()[tombstoneFormatVersionSize:]); err != nil { - return nil, 0, errors.Wrap(err, "write to hash") + return nil, 0, fmt.Errorf("write to hash: %w", err) } if binary.BigEndian.Uint32(b[len(b)-tombstonesCRCSize:]) != hash.Sum32() { return nil, 0, errors.New("checksum did not match") diff --git a/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/dir_locker.go b/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/dir_locker.go index 155f586415..fa939879ca 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/dir_locker.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/dir_locker.go @@ -14,13 +14,13 @@ package tsdbutil import ( + "errors" "fmt" "os" "path/filepath" "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" @@ -83,7 +83,7 @@ func (l *DirLocker) Lock() error { lockf, _, err := fileutil.Flock(l.path) if err != nil { - return errors.Wrap(err, "lock DB directory") + return fmt.Errorf("lock DB directory: %w", err) } l.releaser = lockf return nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/histogram.go b/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/histogram.go index 0327815c48..0847f81a8a 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/histogram.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/tsdbutil/histogram.go @@ -116,7 +116,17 @@ func SetHistogramNotCounterReset(h *histogram.Histogram) *histogram.Histogram { return h } +func SetHistogramCounterReset(h *histogram.Histogram) *histogram.Histogram { + h.CounterResetHint = histogram.CounterReset + return h +} + func SetFloatHistogramNotCounterReset(h *histogram.FloatHistogram) *histogram.FloatHistogram { h.CounterResetHint = histogram.NotCounterReset return h } + +func SetFloatHistogramCounterReset(h *histogram.FloatHistogram) *histogram.FloatHistogram { + h.CounterResetHint = histogram.CounterReset + return h +} diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wal.go b/vendor/github.com/prometheus/prometheus/tsdb/wal.go index 3a410fb636..bc7db35bf1 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wal.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wal.go @@ -11,7 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// nolint:revive // Many unsued function arguments in this file by design. package tsdb import ( @@ -526,14 +525,14 @@ func (w *SegmentWAL) openSegmentFile(name string) (*os.File, error) { case err != nil: return nil, errors.Wrapf(err, "validate meta %q", f.Name()) case n != 8: - return nil, errors.Errorf("invalid header size %d in %q", n, f.Name()) + return nil, fmt.Errorf("invalid header size %d in %q", n, f.Name()) } if m := binary.BigEndian.Uint32(metab[:4]); m != WALMagic { - return nil, errors.Errorf("invalid magic header %x in %q", m, f.Name()) + return nil, fmt.Errorf("invalid magic header %x in %q", m, f.Name()) } if metab[4] != WALFormatDefault { - return nil, errors.Errorf("unknown WAL segment format %d in %q", metab[4], f.Name()) + return nil, fmt.Errorf("unknown WAL segment format %d in %q", metab[4], f.Name()) } hasError = false return f, nil @@ -1053,7 +1052,7 @@ func (e walCorruptionErr) Error() string { func (r *walReader) corruptionErr(s string, args ...interface{}) error { return walCorruptionErr{ - err: errors.Errorf(s, args...), + err: fmt.Errorf(s, args...), file: r.cur, lastOffset: r.lastOffset, } @@ -1125,7 +1124,7 @@ func (r *walReader) decodeSeries(flag byte, b []byte, res *[]record.RefSeries) e return dec.Err() } if len(dec.B) > 0 { - return errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return nil } @@ -1157,7 +1156,7 @@ func (r *walReader) decodeSamples(flag byte, b []byte, res *[]record.RefSample) return errors.Wrapf(dec.Err(), "decode error after %d samples", len(*res)) } if len(dec.B) > 0 { - return errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return nil } @@ -1177,7 +1176,7 @@ func (r *walReader) decodeDeletes(flag byte, b []byte, res *[]tombstones.Stone) return dec.Err() } if len(dec.B) > 0 { - return errors.Errorf("unexpected %d bytes left in entry", len(dec.B)) + return fmt.Errorf("unexpected %d bytes left in entry", len(dec.B)) } return nil } diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wlog/checkpoint.go b/vendor/github.com/prometheus/prometheus/tsdb/wlog/checkpoint.go index d64599c276..3d5b56da27 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wlog/checkpoint.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wlog/checkpoint.go @@ -15,6 +15,7 @@ package wlog import ( + "errors" "fmt" "io" "math" @@ -25,7 +26,6 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/pkg/errors" "golang.org/x/exp/slices" "github.com/prometheus/prometheus/tsdb/chunks" @@ -102,8 +102,8 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head { var sgmRange []SegmentRange dir, idx, err := LastCheckpoint(w.Dir()) - if err != nil && err != record.ErrNotFound { - return nil, errors.Wrap(err, "find last checkpoint") + if err != nil && !errors.Is(err, record.ErrNotFound) { + return nil, fmt.Errorf("find last checkpoint: %w", err) } last := idx + 1 if err == nil { @@ -119,7 +119,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head sgmRange = append(sgmRange, SegmentRange{Dir: w.Dir(), First: from, Last: to}) sgmReader, err = NewSegmentsRangeReader(sgmRange...) if err != nil { - return nil, errors.Wrap(err, "create segment reader") + return nil, fmt.Errorf("create segment reader: %w", err) } defer sgmReader.Close() } @@ -128,15 +128,15 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head cpdirtmp := cpdir + ".tmp" if err := os.RemoveAll(cpdirtmp); err != nil { - return nil, errors.Wrap(err, "remove previous temporary checkpoint dir") + return nil, fmt.Errorf("remove previous temporary checkpoint dir: %w", err) } if err := os.MkdirAll(cpdirtmp, 0o777); err != nil { - return nil, errors.Wrap(err, "create checkpoint dir") + return nil, fmt.Errorf("create checkpoint dir: %w", err) } cp, err := New(nil, nil, cpdirtmp, w.CompressionType()) if err != nil { - return nil, errors.Wrap(err, "open checkpoint") + return nil, fmt.Errorf("open checkpoint: %w", err) } // Ensures that an early return caused by an error doesn't leave any tmp files. @@ -174,7 +174,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head case record.Series: series, err = dec.Series(rec, series) if err != nil { - return nil, errors.Wrap(err, "decode series") + return nil, fmt.Errorf("decode series: %w", err) } // Drop irrelevant series in place. repl := series[:0] @@ -192,7 +192,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head case record.Samples: samples, err = dec.Samples(rec, samples) if err != nil { - return nil, errors.Wrap(err, "decode samples") + return nil, fmt.Errorf("decode samples: %w", err) } // Drop irrelevant samples in place. repl := samples[:0] @@ -210,7 +210,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head case record.HistogramSamples: histogramSamples, err = dec.HistogramSamples(rec, histogramSamples) if err != nil { - return nil, errors.Wrap(err, "decode histogram samples") + return nil, fmt.Errorf("decode histogram samples: %w", err) } // Drop irrelevant histogramSamples in place. repl := histogramSamples[:0] @@ -228,7 +228,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head case record.Tombstones: tstones, err = dec.Tombstones(rec, tstones) if err != nil { - return nil, errors.Wrap(err, "decode deletes") + return nil, fmt.Errorf("decode deletes: %w", err) } // Drop irrelevant tombstones in place. repl := tstones[:0] @@ -249,7 +249,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head case record.Exemplars: exemplars, err = dec.Exemplars(rec, exemplars) if err != nil { - return nil, errors.Wrap(err, "decode exemplars") + return nil, fmt.Errorf("decode exemplars: %w", err) } // Drop irrelevant exemplars in place. repl := exemplars[:0] @@ -266,7 +266,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head case record.Metadata: metadata, err := dec.Metadata(rec, metadata) if err != nil { - return nil, errors.Wrap(err, "decode metadata") + return nil, fmt.Errorf("decode metadata: %w", err) } // Only keep reference to the latest found metadata for each refID. repl := 0 @@ -292,7 +292,7 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head // Flush records in 1 MB increments. if len(buf) > 1*1024*1024 { if err := cp.Log(recs...); err != nil { - return nil, errors.Wrap(err, "flush records") + return nil, fmt.Errorf("flush records: %w", err) } buf, recs = buf[:0], recs[:0] } @@ -300,12 +300,12 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head // If we hit any corruption during checkpointing, repairing is not an option. // The head won't know which series records are lost. if r.Err() != nil { - return nil, errors.Wrap(r.Err(), "read segments") + return nil, fmt.Errorf("read segments: %w", r.Err()) } // Flush remaining records. if err := cp.Log(recs...); err != nil { - return nil, errors.Wrap(err, "flush records") + return nil, fmt.Errorf("flush records: %w", err) } // Flush latest metadata records for each series. @@ -315,29 +315,29 @@ func Checkpoint(logger log.Logger, w *WL, from, to int, keep func(id chunks.Head latestMetadata = append(latestMetadata, m) } if err := cp.Log(enc.Metadata(latestMetadata, buf[:0])); err != nil { - return nil, errors.Wrap(err, "flush metadata records") + return nil, fmt.Errorf("flush metadata records: %w", err) } } if err := cp.Close(); err != nil { - return nil, errors.Wrap(err, "close checkpoint") + return nil, fmt.Errorf("close checkpoint: %w", err) } // Sync temporary directory before rename. df, err := fileutil.OpenDir(cpdirtmp) if err != nil { - return nil, errors.Wrap(err, "open temporary checkpoint directory") + return nil, fmt.Errorf("open temporary checkpoint directory: %w", err) } if err := df.Sync(); err != nil { df.Close() - return nil, errors.Wrap(err, "sync temporary checkpoint directory") + return nil, fmt.Errorf("sync temporary checkpoint directory: %w", err) } if err = df.Close(); err != nil { - return nil, errors.Wrap(err, "close temporary checkpoint directory") + return nil, fmt.Errorf("close temporary checkpoint directory: %w", err) } if err := fileutil.Replace(cpdirtmp, cpdir); err != nil { - return nil, errors.Wrap(err, "rename checkpoint directory") + return nil, fmt.Errorf("rename checkpoint directory: %w", err) } return stats, nil @@ -364,7 +364,7 @@ func listCheckpoints(dir string) (refs []checkpointRef, err error) { continue } if !fi.IsDir() { - return nil, errors.Errorf("checkpoint %s is not a directory", fi.Name()) + return nil, fmt.Errorf("checkpoint %s is not a directory", fi.Name()) } idx, err := strconv.Atoi(fi.Name()[len(checkpointPrefix):]) if err != nil { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wlog/live_reader.go b/vendor/github.com/prometheus/prometheus/tsdb/wlog/live_reader.go index c69017051b..905bbf00d6 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wlog/live_reader.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wlog/live_reader.go @@ -16,6 +16,7 @@ package wlog import ( "encoding/binary" + "errors" "fmt" "hash/crc32" "io" @@ -24,7 +25,6 @@ import ( "github.com/go-kit/log/level" "github.com/golang/snappy" "github.com/klauspost/compress/zstd" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" ) @@ -135,7 +135,7 @@ func (r *LiveReader) Next() bool { switch ok, err := r.buildRecord(); { case ok: return true - case err != nil && err != io.EOF: + case err != nil && !errors.Is(err, io.EOF): r.err = err return false } @@ -157,7 +157,7 @@ func (r *LiveReader) Next() bool { if r.writeIndex != pageSize { n, err := r.fillBuffer() - if n == 0 || (err != nil && err != io.EOF) { + if n == 0 || (err != nil && !errors.Is(err, io.EOF)) { r.err = err return false } @@ -174,7 +174,7 @@ func (r *LiveReader) Record() []byte { // Rebuild a full record from potentially partial records. Returns false // if there was an error or if we weren't able to read a record for any reason. // Returns true if we read a full record. Any record data is appended to -// LiveReader.rec +// LiveReader.rec. func (r *LiveReader) buildRecord() (bool, error) { for { // Check that we have data in the internal buffer to read. @@ -265,7 +265,7 @@ func validateRecord(typ recType, i int) error { } return nil default: - return errors.Errorf("unexpected record type %d", typ) + return fmt.Errorf("unexpected record type %d", typ) } } @@ -322,7 +322,7 @@ func (r *LiveReader) readRecord() ([]byte, int, error) { rec := r.buf[r.readIndex+recordHeaderSize : r.readIndex+recordHeaderSize+length] if c := crc32.Checksum(rec, castagnoliTable); c != crc { - return nil, 0, errors.Errorf("unexpected checksum %x, expected %x", c, crc) + return nil, 0, fmt.Errorf("unexpected checksum %x, expected %x", c, crc) } return rec, length + recordHeaderSize, nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wlog/reader.go b/vendor/github.com/prometheus/prometheus/tsdb/wlog/reader.go index f77b03b8ea..a744b0cc4b 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wlog/reader.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wlog/reader.go @@ -16,12 +16,13 @@ package wlog import ( "encoding/binary" + "errors" + "fmt" "hash/crc32" "io" "github.com/golang/snappy" "github.com/klauspost/compress/zstd" - "github.com/pkg/errors" ) // Reader reads WAL records from an io.Reader. @@ -47,7 +48,7 @@ func NewReader(r io.Reader) *Reader { // It must not be called again after it returned false. func (r *Reader) Next() bool { err := r.next() - if errors.Is(err, io.EOF) { + if err != nil && errors.Is(err, io.EOF) { // The last WAL segment record shouldn't be torn(should be full or last). // The last record would be torn after a crash just before // the last record part could be persisted to disk. @@ -72,7 +73,7 @@ func (r *Reader) next() (err error) { i := 0 for { if _, err = io.ReadFull(r.rdr, hdr[:1]); err != nil { - return errors.Wrap(err, "read first header byte") + return fmt.Errorf("read first header byte: %w", err) } r.total++ r.curRecTyp = recTypeFromHeader(hdr[0]) @@ -95,7 +96,7 @@ func (r *Reader) next() (err error) { } n, err := io.ReadFull(r.rdr, buf[:k]) if err != nil { - return errors.Wrap(err, "read remaining zeros") + return fmt.Errorf("read remaining zeros: %w", err) } r.total += int64(n) @@ -108,7 +109,7 @@ func (r *Reader) next() (err error) { } n, err := io.ReadFull(r.rdr, hdr[1:]) if err != nil { - return errors.Wrap(err, "read remaining header") + return fmt.Errorf("read remaining header: %w", err) } r.total += int64(n) @@ -118,7 +119,7 @@ func (r *Reader) next() (err error) { ) if length > pageSize-recordHeaderSize { - return errors.Errorf("invalid record size %d", length) + return fmt.Errorf("invalid record size %d", length) } n, err = io.ReadFull(r.rdr, buf[:length]) if err != nil { @@ -127,10 +128,10 @@ func (r *Reader) next() (err error) { r.total += int64(n) if n != int(length) { - return errors.Errorf("invalid size: expected %d, got %d", length, n) + return fmt.Errorf("invalid size: expected %d, got %d", length, n) } if c := crc32.Checksum(buf[:length], castagnoliTable); c != crc { - return errors.Errorf("unexpected checksum %x, expected %x", c, crc) + return fmt.Errorf("unexpected checksum %x, expected %x", c, crc) } if isSnappyCompressed || isZstdCompressed { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wlog/watcher.go b/vendor/github.com/prometheus/prometheus/tsdb/wlog/watcher.go index 221e9607ca..c9f8a4599f 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wlog/watcher.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wlog/watcher.go @@ -65,7 +65,7 @@ type WriteTo interface { SeriesReset(int) } -// Used to notifier the watcher that data has been written so that it can read. +// Used to notify the watcher that data has been written so that it can read. type WriteNotified interface { Notify() } @@ -398,28 +398,26 @@ func (w *Watcher) watch(segmentNum int, tail bool) error { reader := NewLiveReader(w.logger, w.readerMetrics, segment) - readTicker := time.NewTicker(readTimeout) - defer readTicker.Stop() - - checkpointTicker := time.NewTicker(checkpointPeriod) - defer checkpointTicker.Stop() - - segmentTicker := time.NewTicker(segmentCheckPeriod) - defer segmentTicker.Stop() - - // If we're replaying the segment we need to know the size of the file to know - // when to return from watch and move on to the next segment. size := int64(math.MaxInt64) if !tail { - segmentTicker.Stop() - checkpointTicker.Stop() var err error size, err = getSegmentSize(w.walDir, segmentNum) if err != nil { return errors.Wrap(err, "getSegmentSize") } + + return w.readAndHandleError(reader, segmentNum, tail, size) } + checkpointTicker := time.NewTicker(checkpointPeriod) + defer checkpointTicker.Stop() + + segmentTicker := time.NewTicker(segmentCheckPeriod) + defer segmentTicker.Stop() + + readTicker := time.NewTicker(readTimeout) + defer readTicker.Stop() + gcSem := make(chan struct{}, 1) for { select { @@ -747,12 +745,12 @@ func checkpointNum(dir string) (int, error) { // dir may contain a hidden directory, so only check the base directory chunks := strings.Split(filepath.Base(dir), ".") if len(chunks) != 2 { - return 0, errors.Errorf("invalid checkpoint dir string: %s", dir) + return 0, fmt.Errorf("invalid checkpoint dir string: %s", dir) } result, err := strconv.Atoi(chunks[1]) if err != nil { - return 0, errors.Errorf("invalid checkpoint dir string: %s", dir) + return 0, fmt.Errorf("invalid checkpoint dir string: %s", dir) } return result, nil diff --git a/vendor/github.com/prometheus/prometheus/tsdb/wlog/wlog.go b/vendor/github.com/prometheus/prometheus/tsdb/wlog/wlog.go index fd65fca07a..c3ae001d98 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/wlog/wlog.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/wlog/wlog.go @@ -17,6 +17,7 @@ package wlog import ( "bufio" "encoding/binary" + "errors" "fmt" "hash/crc32" "io" @@ -30,7 +31,6 @@ import ( "github.com/go-kit/log/level" "github.com/golang/snappy" "github.com/klauspost/compress/zstd" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "golang.org/x/exp/slices" @@ -137,7 +137,7 @@ func OpenWriteSegment(logger log.Logger, dir string, k int) (*Segment, error) { level.Warn(logger).Log("msg", "Last page of the wlog is torn, filling it with zeros", "segment", segName) if _, err := f.Write(make([]byte, pageSize-d)); err != nil { f.Close() - return nil, errors.Wrap(err, "zero-pad torn page") + return nil, fmt.Errorf("zero-pad torn page: %w", err) } } return &Segment{SegmentFile: f, i: k, dir: dir}, nil @@ -298,7 +298,7 @@ func NewSize(logger log.Logger, reg prometheus.Registerer, dir string, segmentSi return nil, errors.New("invalid segment size") } if err := os.MkdirAll(dir, 0o777); err != nil { - return nil, errors.Wrap(err, "create dir") + return nil, fmt.Errorf("create dir: %w", err) } if logger == nil { logger = log.NewNopLogger() @@ -331,7 +331,7 @@ func NewSize(logger log.Logger, reg prometheus.Registerer, dir string, segmentSi _, last, err := Segments(w.Dir()) if err != nil { - return nil, errors.Wrap(err, "get segment range") + return nil, fmt.Errorf("get segment range: %w", err) } // Index of the Segment we want to open and write to. @@ -414,11 +414,9 @@ func (w *WL) Repair(origErr error) error { // But that's not generally applicable if the records have any kind of causality. // Maybe as an extra mode in the future if mid-WAL corruptions become // a frequent concern. - err := errors.Cause(origErr) // So that we can pick up errors even if wrapped. - - cerr, ok := err.(*CorruptionErr) - if !ok { - return errors.Wrap(origErr, "cannot handle error") + var cerr *CorruptionErr + if !errors.As(origErr, &cerr) { + return fmt.Errorf("cannot handle error: %w", origErr) } if cerr.Segment < 0 { return errors.New("corruption error does not specify position") @@ -429,7 +427,7 @@ func (w *WL) Repair(origErr error) error { // All segments behind the corruption can no longer be used. segs, err := listSegments(w.Dir()) if err != nil { - return errors.Wrap(err, "list segments") + return fmt.Errorf("list segments: %w", err) } level.Warn(w.logger).Log("msg", "Deleting all segments newer than corrupted segment", "segment", cerr.Segment) @@ -440,14 +438,14 @@ func (w *WL) Repair(origErr error) error { // as we set the current segment to repaired file // below. if err := w.segment.Close(); err != nil { - return errors.Wrap(err, "close active segment") + return fmt.Errorf("close active segment: %w", err) } } if s.index <= cerr.Segment { continue } if err := os.Remove(filepath.Join(w.Dir(), s.name)); err != nil { - return errors.Wrapf(err, "delete segment:%v", s.index) + return fmt.Errorf("delete segment:%v: %w", s.index, err) } } // Regardless of the corruption offset, no record reaches into the previous segment. @@ -472,7 +470,7 @@ func (w *WL) Repair(origErr error) error { f, err := os.Open(tmpfn) if err != nil { - return errors.Wrap(err, "open segment") + return fmt.Errorf("open segment: %w", err) } defer f.Close() @@ -484,24 +482,24 @@ func (w *WL) Repair(origErr error) error { break } if err := w.Log(r.Record()); err != nil { - return errors.Wrap(err, "insert record") + return fmt.Errorf("insert record: %w", err) } } // We expect an error here from r.Err(), so nothing to handle. // We need to pad to the end of the last page in the repaired segment if err := w.flushPage(true); err != nil { - return errors.Wrap(err, "flush page in repair") + return fmt.Errorf("flush page in repair: %w", err) } // We explicitly close even when there is a defer for Windows to be // able to delete it. The defer is in place to close it in-case there // are errors above. if err := f.Close(); err != nil { - return errors.Wrap(err, "close corrupted file") + return fmt.Errorf("close corrupted file: %w", err) } if err := os.Remove(tmpfn); err != nil { - return errors.Wrap(err, "delete corrupted segment") + return fmt.Errorf("delete corrupted segment: %w", err) } // Explicitly close the segment we just repaired to avoid issues with Windows. @@ -553,7 +551,7 @@ func (w *WL) nextSegment(async bool) (int, error) { } next, err := CreateSegment(w.Dir(), w.segment.Index()+1) if err != nil { - return 0, errors.Wrap(err, "create new segment file") + return 0, fmt.Errorf("create new segment file: %w", err) } prev := w.segment if err := w.setSegment(next); err != nil { @@ -622,7 +620,8 @@ func (w *WL) flushPage(clear bool) error { } // First Byte of header format: -// [3 bits unallocated] [1 bit zstd compression flag] [1 bit snappy compression flag] [3 bit record type ] +// +// [3 bits unallocated] [1 bit zstd compression flag] [1 bit snappy compression flag] [3 bit record type ] const ( snappyMask = 1 << 3 zstdMask = 1 << 4 @@ -836,7 +835,7 @@ func (w *WL) fsync(f *Segment) error { // Sync forces a file sync on the current write log segment. This function is meant // to be used only on tests due to different behaviour on Operating Systems -// like windows and linux +// like windows and linux. func (w *WL) Sync() error { return w.fsync(w.segment) } @@ -939,7 +938,7 @@ func NewSegmentsRangeReader(sr ...SegmentRange) (io.ReadCloser, error) { for _, sgmRange := range sr { refs, err := listSegments(sgmRange.Dir) if err != nil { - return nil, errors.Wrapf(err, "list segment in dir:%v", sgmRange.Dir) + return nil, fmt.Errorf("list segment in dir:%v: %w", sgmRange.Dir, err) } for _, r := range refs { @@ -951,7 +950,7 @@ func NewSegmentsRangeReader(sr ...SegmentRange) (io.ReadCloser, error) { } s, err := OpenReadSegment(filepath.Join(sgmRange.Dir, r.name)) if err != nil { - return nil, errors.Wrapf(err, "open segment:%v in dir:%v", r.name, sgmRange.Dir) + return nil, fmt.Errorf("open segment:%v in dir:%v: %w", r.name, sgmRange.Dir, err) } segs = append(segs, s) } @@ -971,8 +970,7 @@ type segmentBufReader struct { off int // Offset of read data into current segment. } -// nolint:revive // TODO: Consider exporting segmentBufReader -func NewSegmentBufReader(segs ...*Segment) *segmentBufReader { +func NewSegmentBufReader(segs ...*Segment) io.ReadCloser { if len(segs) == 0 { return &segmentBufReader{} } @@ -983,16 +981,16 @@ func NewSegmentBufReader(segs ...*Segment) *segmentBufReader { } } -// nolint:revive -func NewSegmentBufReaderWithOffset(offset int, segs ...*Segment) (sbr *segmentBufReader, err error) { +func NewSegmentBufReaderWithOffset(offset int, segs ...*Segment) (io.ReadCloser, error) { if offset == 0 || len(segs) == 0 { return NewSegmentBufReader(segs...), nil } - sbr = &segmentBufReader{ + sbr := &segmentBufReader{ buf: bufio.NewReaderSize(segs[0], 16*pageSize), segs: segs, } + var err error if offset > 0 { _, err = sbr.buf.Discard(offset) } @@ -1018,7 +1016,7 @@ func (r *segmentBufReader) Read(b []byte) (n int, err error) { r.off += n // If we succeeded, or hit a non-EOF, we can stop. - if err == nil || err != io.EOF { + if err == nil || !errors.Is(err, io.EOF) { return n, err } diff --git a/vendor/github.com/prometheus/prometheus/util/annotations/annotations.go b/vendor/github.com/prometheus/prometheus/util/annotations/annotations.go index 1d339646e6..180d408430 100644 --- a/vendor/github.com/prometheus/prometheus/util/annotations/annotations.go +++ b/vendor/github.com/prometheus/prometheus/util/annotations/annotations.go @@ -81,8 +81,8 @@ func (a Annotations) AsStrings(query string, maxAnnos int) []string { if maxAnnos > 0 && len(arr) >= maxAnnos { break } - anErr, ok := err.(annoErr) - if ok { + var anErr annoErr + if errors.As(err, &anErr) { anErr.Query = query err = anErr } @@ -94,7 +94,7 @@ func (a Annotations) AsStrings(query string, maxAnnos int) []string { return arr } -//nolint:revive // Ignore ST1012 +//nolint:revive // error-naming. var ( // Currently there are only 2 types, warnings and info. // For now, info are visually identical with warnings as we have not updated @@ -109,7 +109,7 @@ var ( MixedClassicNativeHistogramsWarning = fmt.Errorf("%w: vector contains a mix of classic and native histograms for metric name", PromQLWarning) PossibleNonCounterInfo = fmt.Errorf("%w: metric might not be a counter, name does not end in _total/_sum/_count/_bucket:", PromQLInfo) - HistogramQuantileForcedMonotonicityInfo = fmt.Errorf("%w: input to histogram_quantile needed to be fixed for monotonicity (and may give inaccurate results) for metric name", PromQLInfo) + HistogramQuantileForcedMonotonicityInfo = fmt.Errorf("%w: input to histogram_quantile needed to be fixed for monotonicity (see https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile) for metric name", PromQLInfo) ) type annoErr struct { @@ -119,12 +119,19 @@ type annoErr struct { } func (e annoErr) Error() string { + if e.Query == "" { + return e.Err.Error() + } return fmt.Sprintf("%s (%s)", e.Err, e.PositionRange.StartPosInput(e.Query, 0)) } +func (e annoErr) Unwrap() error { + return e.Err +} + // NewInvalidQuantileWarning is used when the user specifies an invalid quantile // value, i.e. a float that is outside the range [0, 1] or NaN. -func NewInvalidQuantileWarning(q float64, pos posrange.PositionRange) annoErr { +func NewInvalidQuantileWarning(q float64, pos posrange.PositionRange) error { return annoErr{ PositionRange: pos, Err: fmt.Errorf("%w, got %g", InvalidQuantileWarning, q), @@ -133,7 +140,7 @@ func NewInvalidQuantileWarning(q float64, pos posrange.PositionRange) annoErr { // NewBadBucketLabelWarning is used when there is an error parsing the bucket label // of a classic histogram. -func NewBadBucketLabelWarning(metricName, label string, pos posrange.PositionRange) annoErr { +func NewBadBucketLabelWarning(metricName, label string, pos posrange.PositionRange) error { return annoErr{ PositionRange: pos, Err: fmt.Errorf("%w of %q for metric name %q", BadBucketLabelWarning, label, metricName), @@ -143,7 +150,7 @@ func NewBadBucketLabelWarning(metricName, label string, pos posrange.PositionRan // NewMixedFloatsHistogramsWarning is used when the queried series includes both // float samples and histogram samples for functions that do not support mixed // samples. -func NewMixedFloatsHistogramsWarning(metricName string, pos posrange.PositionRange) annoErr { +func NewMixedFloatsHistogramsWarning(metricName string, pos posrange.PositionRange) error { return annoErr{ PositionRange: pos, Err: fmt.Errorf("%w %q", MixedFloatsHistogramsWarning, metricName), @@ -152,7 +159,7 @@ func NewMixedFloatsHistogramsWarning(metricName string, pos posrange.PositionRan // NewMixedClassicNativeHistogramsWarning is used when the queried series includes // both classic and native histograms. -func NewMixedClassicNativeHistogramsWarning(metricName string, pos posrange.PositionRange) annoErr { +func NewMixedClassicNativeHistogramsWarning(metricName string, pos posrange.PositionRange) error { return annoErr{ PositionRange: pos, Err: fmt.Errorf("%w %q", MixedClassicNativeHistogramsWarning, metricName), @@ -161,7 +168,7 @@ func NewMixedClassicNativeHistogramsWarning(metricName string, pos posrange.Posi // NewPossibleNonCounterInfo is used when a named counter metric with only float samples does not // have the suffixes _total, _sum, _count, or _bucket. -func NewPossibleNonCounterInfo(metricName string, pos posrange.PositionRange) annoErr { +func NewPossibleNonCounterInfo(metricName string, pos posrange.PositionRange) error { return annoErr{ PositionRange: pos, Err: fmt.Errorf("%w %q", PossibleNonCounterInfo, metricName), @@ -170,7 +177,7 @@ func NewPossibleNonCounterInfo(metricName string, pos posrange.PositionRange) an // NewHistogramQuantileForcedMonotonicityInfo is used when the input (classic histograms) to // histogram_quantile needs to be forced to be monotonic. -func NewHistogramQuantileForcedMonotonicityInfo(metricName string, pos posrange.PositionRange) annoErr { +func NewHistogramQuantileForcedMonotonicityInfo(metricName string, pos posrange.PositionRange) error { return annoErr{ PositionRange: pos, Err: fmt.Errorf("%w %q", HistogramQuantileForcedMonotonicityInfo, metricName), diff --git a/vendor/github.com/prometheus/prometheus/util/testutil/context.go b/vendor/github.com/prometheus/prometheus/util/testutil/context.go index c1f4a831ce..3f63b030d7 100644 --- a/vendor/github.com/prometheus/prometheus/util/testutil/context.go +++ b/vendor/github.com/prometheus/prometheus/util/testutil/context.go @@ -15,18 +15,18 @@ package testutil import "time" -// A MockContext provides a simple stub implementation of a Context +// A MockContext provides a simple stub implementation of a Context. type MockContext struct { Error error DoneCh chan struct{} } -// Deadline always will return not set +// Deadline always will return not set. func (c *MockContext) Deadline() (deadline time.Time, ok bool) { return time.Time{}, false } -// Done returns a read channel for listening to the Done event +// Done returns a read channel for listening to the Done event. func (c *MockContext) Done() <-chan struct{} { return c.DoneCh } @@ -36,7 +36,7 @@ func (c *MockContext) Err() error { return c.Error } -// Value ignores the Value and always returns nil +// Value ignores the Value and always returns nil. func (c *MockContext) Value(interface{}) interface{} { return nil } diff --git a/vendor/github.com/prometheus/prometheus/util/testutil/directory.go b/vendor/github.com/prometheus/prometheus/util/testutil/directory.go index 71ed74ba55..8aa17702d2 100644 --- a/vendor/github.com/prometheus/prometheus/util/testutil/directory.go +++ b/vendor/github.com/prometheus/prometheus/util/testutil/directory.go @@ -33,7 +33,7 @@ const ( // NilCloser is a no-op Closer. NilCloser = nilCloser(true) - // The number of times that a TemporaryDirectory will retry its removal + // The number of times that a TemporaryDirectory will retry its removal. temporaryDirectoryRemoveRetries = 2 ) diff --git a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go index 1a54f23a61..671df78872 100644 --- a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go +++ b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go @@ -15,6 +15,7 @@ package v1 import ( "context" + "errors" "fmt" "math" "math/rand" @@ -33,7 +34,6 @@ import ( "github.com/grafana/regexp" jsoniter "github.com/json-iterator/go" "github.com/munnerz/goautoneg" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/common/route" @@ -317,7 +317,7 @@ func (api *API) ClearCodecs() { } func setUnavailStatusOnTSDBNotReady(r apiFuncResult) apiFuncResult { - if r.err != nil && errors.Cause(r.err.err) == tsdb.ErrNotReady { + if r.err != nil && errors.Is(r.err.err, tsdb.ErrNotReady) { r.err.typ = errorUnavailable } return r @@ -415,7 +415,7 @@ type QueryData struct { func invalidParamError(err error, parameter string) apiFuncResult { return apiFuncResult{nil, &apiError{ - errorBadData, errors.Wrapf(err, "invalid parameter %q", parameter), + errorBadData, fmt.Errorf("invalid parameter %q: %w", parameter, err), }, nil, nil} } @@ -624,17 +624,15 @@ func returnAPIError(err error) *apiError { return nil } - cause := errors.Unwrap(err) - if cause == nil { - cause = err - } - - switch cause.(type) { - case promql.ErrQueryCanceled: + var eqc promql.ErrQueryCanceled + var eqt promql.ErrQueryTimeout + var es promql.ErrStorage + switch { + case errors.As(err, &eqc): return &apiError{errorCanceled, err} - case promql.ErrQueryTimeout: + case errors.As(err, &eqt): return &apiError{errorTimeout, err} - case promql.ErrStorage: + case errors.As(err, &es): return &apiError{errorInternal, err} } @@ -670,7 +668,7 @@ func (api *API) labelNames(r *http.Request) apiFuncResult { names []string warnings annotations.Annotations ) - if len(matcherSets) > 0 { + if len(matcherSets) > 1 { labelNamesSet := make(map[string]struct{}) for _, matchers := range matcherSets { @@ -692,7 +690,11 @@ func (api *API) labelNames(r *http.Request) apiFuncResult { } slices.Sort(names) } else { - names, warnings, err = q.LabelNames(r.Context()) + var matchers []*labels.Matcher + if len(matcherSets) == 1 { + matchers = matcherSets[0] + } + names, warnings, err = q.LabelNames(r.Context(), matchers...) if err != nil { return apiFuncResult{nil, &apiError{errorExec, err}, warnings, nil} } @@ -709,7 +711,7 @@ func (api *API) labelValues(r *http.Request) (result apiFuncResult) { name := route.Param(ctx, "name") if !model.LabelNameRE.MatchString(name) { - return apiFuncResult{nil, &apiError{errorBadData, errors.Errorf("invalid label name: %q", name)}, nil, nil} + return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("invalid label name: %q", name)}, nil, nil} } start, err := parseTimeParam(r, "start", MinTime) @@ -746,7 +748,7 @@ func (api *API) labelValues(r *http.Request) (result apiFuncResult) { vals []string warnings annotations.Annotations ) - if len(matcherSets) > 0 { + if len(matcherSets) > 1 { var callWarnings annotations.Annotations labelValuesSet := make(map[string]struct{}) for _, matchers := range matcherSets { @@ -765,7 +767,11 @@ func (api *API) labelValues(r *http.Request) (result apiFuncResult) { vals = append(vals, val) } } else { - vals, warnings, err = q.LabelValues(ctx, name) + var matchers []*labels.Matcher + if len(matcherSets) == 1 { + matchers = matcherSets[0] + } + vals, warnings, err = q.LabelValues(ctx, name, matchers...) if err != nil { return apiFuncResult{nil, &apiError{errorExec, err}, warnings, closer} } @@ -797,7 +803,7 @@ func (api *API) series(r *http.Request) (result apiFuncResult) { ctx := r.Context() if err := r.ParseForm(); err != nil { - return apiFuncResult{nil, &apiError{errorBadData, errors.Wrapf(err, "error parsing form values")}, nil, nil} + return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing form values: %w", err)}, nil, nil} } if len(r.Form["match[]"]) == 0 { return apiFuncResult{nil, &apiError{errorBadData, errors.New("no match[] parameter provided")}, nil, nil} @@ -1028,7 +1034,7 @@ func (api *API) targets(r *http.Request) apiFuncResult { case err == nil && lastErrStr == "": return "" case err != nil: - return errors.Wrapf(err, lastErrStr).Error() + return fmt.Errorf("%s: %w", lastErrStr, err).Error() default: return lastErrStr } @@ -1294,12 +1300,12 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult { return apiFuncResult{res, nil, nil, nil} } -// RuleDiscovery has info for all rules +// RuleDiscovery has info for all rules. type RuleDiscovery struct { RuleGroups []*RuleGroup `json:"groups"` } -// RuleGroup has info for rules which are part of a group +// RuleGroup has info for rules which are part of a group. type RuleGroup struct { Name string `json:"name"` File string `json:"file"` @@ -1347,7 +1353,7 @@ type RecordingRule struct { func (api *API) rules(r *http.Request) apiFuncResult { if err := r.ParseForm(); err != nil { - return apiFuncResult{nil, &apiError{errorBadData, errors.Wrapf(err, "error parsing form values")}, nil, nil} + return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing form values: %w", err)}, nil, nil} } queryFormToSet := func(values []string) map[string]struct{} { @@ -1367,12 +1373,17 @@ func (api *API) rules(r *http.Request) apiFuncResult { typ := strings.ToLower(r.URL.Query().Get("type")) if typ != "" && typ != "alert" && typ != "record" { - return invalidParamError(errors.Errorf("not supported value %q", typ), "type") + return invalidParamError(fmt.Errorf("not supported value %q", typ), "type") } returnAlerts := typ == "" || typ == "alert" returnRecording := typ == "" || typ == "record" + excludeAlerts, err := parseExcludeAlerts(r) + if err != nil { + return invalidParamError(err, "exclude_alerts") + } + rgs := make([]*RuleGroup, 0, len(ruleGroups)) for _, grp := range ruleGroups { if len(rgSet) > 0 { @@ -1414,6 +1425,10 @@ func (api *API) rules(r *http.Request) apiFuncResult { if !returnAlerts { break } + var activeAlerts []*Alert + if !excludeAlerts { + activeAlerts = rulesAlertsToAPIAlerts(rule.ActiveAlerts()) + } enrichedRule = AlertingRule{ State: rule.State().String(), Name: rule.Name(), @@ -1422,7 +1437,7 @@ func (api *API) rules(r *http.Request) apiFuncResult { KeepFiringFor: rule.KeepFiringFor().Seconds(), Labels: rule.Labels(), Annotations: rule.Annotations(), - Alerts: rulesAlertsToAPIAlerts(rule.ActiveAlerts()), + Alerts: activeAlerts, Health: rule.Health(), LastError: lastError, EvaluationTime: rule.GetEvaluationDuration().Seconds(), @@ -1444,7 +1459,7 @@ func (api *API) rules(r *http.Request) apiFuncResult { Type: "recording", } default: - err := errors.Errorf("failed to assert type of rule '%v'", rule.Name()) + err := fmt.Errorf("failed to assert type of rule '%v'", rule.Name()) return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil} } @@ -1462,6 +1477,20 @@ func (api *API) rules(r *http.Request) apiFuncResult { return apiFuncResult{res, nil, nil, nil} } +func parseExcludeAlerts(r *http.Request) (bool, error) { + excludeAlertsParam := strings.ToLower(r.URL.Query().Get("exclude_alerts")) + + if excludeAlertsParam == "" { + return false, nil + } + + excludeAlerts, err := strconv.ParseBool(excludeAlertsParam) + if err != nil { + return false, fmt.Errorf("error converting exclude_alerts: %w", err) + } + return excludeAlerts, nil +} + type prometheusConfig struct { YAML string `json:"yaml"` } @@ -1537,7 +1566,7 @@ func (api *API) serveTSDBStatus(r *http.Request) apiFuncResult { } metrics, err := api.gatherer.Gather() if err != nil { - return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("error gathering runtime status: %s", err)}, nil, nil} + return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("error gathering runtime status: %w", err)}, nil, nil} } chunkCount := int64(math.NaN()) for _, mF := range metrics { @@ -1613,7 +1642,7 @@ func (api *API) deleteSeries(r *http.Request) apiFuncResult { return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("admin APIs disabled")}, nil, nil} } if err := r.ParseForm(); err != nil { - return apiFuncResult{nil, &apiError{errorBadData, errors.Wrap(err, "error parsing form values")}, nil, nil} + return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing form values: %w", err)}, nil, nil} } if len(r.Form["match[]"]) == 0 { return apiFuncResult{nil, &apiError{errorBadData, errors.New("no match[] parameter provided")}, nil, nil} @@ -1652,7 +1681,7 @@ func (api *API) snapshot(r *http.Request) apiFuncResult { if r.FormValue("skip_head") != "" { skipHead, err = strconv.ParseBool(r.FormValue("skip_head")) if err != nil { - return invalidParamError(errors.Wrapf(err, "unable to parse boolean"), "skip_head") + return invalidParamError(fmt.Errorf("unable to parse boolean: %w", err), "skip_head") } } @@ -1664,10 +1693,10 @@ func (api *API) snapshot(r *http.Request) apiFuncResult { dir = filepath.Join(snapdir, name) ) if err := os.MkdirAll(dir, 0o777); err != nil { - return apiFuncResult{nil, &apiError{errorInternal, errors.Wrap(err, "create snapshot directory")}, nil, nil} + return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("create snapshot directory: %w", err)}, nil, nil} } if err := api.db.Snapshot(dir, !skipHead); err != nil { - return apiFuncResult{nil, &apiError{errorInternal, errors.Wrap(err, "create snapshot")}, nil, nil} + return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("create snapshot: %w", err)}, nil, nil} } return apiFuncResult{struct { @@ -1782,7 +1811,7 @@ func parseTimeParam(r *http.Request, paramName string, defaultValue time.Time) ( } result, err := parseTime(val) if err != nil { - return time.Time{}, errors.Wrapf(err, "Invalid time value for '%s'", paramName) + return time.Time{}, fmt.Errorf("Invalid time value for '%s': %w", paramName, err) } return result, nil } @@ -1807,21 +1836,21 @@ func parseTime(s string) (time.Time, error) { case maxTimeFormatted: return MaxTime, nil } - return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s) + return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s) } func parseDuration(s string) (time.Duration, error) { if d, err := strconv.ParseFloat(s, 64); err == nil { ts := d * float64(time.Second) if ts > float64(math.MaxInt64) || ts < float64(math.MinInt64) { - return 0, errors.Errorf("cannot parse %q to a valid duration. It overflows int64", s) + return 0, fmt.Errorf("cannot parse %q to a valid duration. It overflows int64", s) } return time.Duration(ts), nil } if d, err := model.ParseDuration(s); err == nil { return time.Duration(d), nil } - return 0, errors.Errorf("cannot parse %q to a valid duration", s) + return 0, fmt.Errorf("cannot parse %q to a valid duration", s) } func parseMatchersParam(matchers []string) ([][]*labels.Matcher, error) { diff --git a/vendor/github.com/thanos-io/thanos/pkg/block/index.go b/vendor/github.com/thanos-io/thanos/pkg/block/index.go index 170f6fbed4..7d4c876154 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/block/index.go +++ b/vendor/github.com/thanos-io/thanos/pkg/block/index.go @@ -616,7 +616,8 @@ func rewrite( builder.Sort() for i, c := range chks { - chks[i].Chunk, err = chunkr.Chunk(c) + // Ignore iterable as it should be nil. + chks[i].Chunk, _, err = chunkr.ChunkOrIterable(c) if err != nil { return errors.Wrap(err, "chunk read") } diff --git a/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go b/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go index acc3f4132f..c86185bfbf 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go +++ b/vendor/github.com/thanos-io/thanos/pkg/block/indexheader/binary_reader.go @@ -983,7 +983,7 @@ func (r *BinaryReader) LookupSymbol(ctx context.Context, o uint32) (string, erro } r.valueSymbolsMx.RUnlock() - s, err := r.symbols.Lookup(ctx, o) + s, err := r.symbols.Lookup(o) if err != nil { return s, err } diff --git a/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go b/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go index 14963602e6..c34f598354 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go +++ b/vendor/github.com/thanos-io/thanos/pkg/compact/downsample/downsample.go @@ -138,7 +138,8 @@ func Downsample( // While #183 exists, we sanitize the chunks we retrieved from the block // before retrieving their samples. for i, c := range chks { - chk, err := chunkr.Chunk(c) + // Ignore iterable as it should be nil. + chk, _, err := chunkr.ChunkOrIterable(c) if err != nil { return id, errors.Wrapf(err, "get chunk %d, series %d", c.Ref, postings.At()) } diff --git a/vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go b/vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go index 40bce8a36a..b7ab4988f4 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go +++ b/vendor/github.com/thanos-io/thanos/pkg/errutil/multierror.go @@ -47,7 +47,7 @@ func (es *SyncMultiError) Add(err error) { es.mtx.Lock() defer es.mtx.Unlock() - es.Add(err) + es.es.Add(err) } // Err returns the error list as an error or nil if it is empty. diff --git a/vendor/github.com/thanos-io/thanos/pkg/httpconfig/http.go b/vendor/github.com/thanos-io/thanos/pkg/httpconfig/http.go index 86d62780f0..4b71a03fda 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/httpconfig/http.go +++ b/vendor/github.com/thanos-io/thanos/pkg/httpconfig/http.go @@ -163,7 +163,8 @@ func NewRoundTripperFromConfig(cfg config_util.HTTPClientConfig, transportConfig } if cfg.BasicAuth != nil { - rt = config_util.NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.PasswordFile, rt) + // TODO(yeya24): expose UsernameFile as a config. + rt = config_util.NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, "", cfg.BasicAuth.PasswordFile, rt) } // Return a new configured RoundTripper. return rt, nil diff --git a/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go b/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go index ab8665f451..5c266c83ee 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go +++ b/vendor/github.com/thanos-io/thanos/pkg/runutil/runutil.go @@ -135,8 +135,7 @@ func ExhaustCloseWithLogOnErr(logger log.Logger, r io.ReadCloser, format string, CloseWithLogOnErr(logger, r, format, a...) } -// CloseWithErrCapture runs function and on error return error by argument including the given error (usually -// from caller function). +// CloseWithErrCapture closes closer, wraps any error with message from fmt and args, and stores this in err. func CloseWithErrCapture(err *error, closer io.Closer, format string, a ...interface{}) { merr := errutil.MultiError{} diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go index 93a7a50892..da81fab93a 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go @@ -1726,7 +1726,7 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq indexr := b.indexReader() g.Go(func() error { - span, newCtx := tracing.StartSpan(gctx, "bucket_store_block_series", tracing.Tags{ + span, newCtx := tracing.StartSpan(gctx, "bucket_store_block_label_names", tracing.Tags{ "block.id": b.meta.ULID, "block.mint": b.meta.MinTime, "block.maxt": b.meta.MaxTime, @@ -1934,7 +1934,7 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR indexr := b.indexReader() g.Go(func() error { - span, newCtx := tracing.StartSpan(gctx, "bucket_store_block_series", tracing.Tags{ + span, newCtx := tracing.StartSpan(gctx, "bucket_store_block_label_values", tracing.Tags{ "block.id": b.meta.ULID, "block.mint": b.meta.MinTime, "block.maxt": b.meta.MaxTime, diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/cache/filter_cache.go b/vendor/github.com/thanos-io/thanos/pkg/store/cache/filter_cache.go index 994491595f..44d46db709 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/cache/filter_cache.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/cache/filter_cache.go @@ -14,22 +14,25 @@ import ( ) type FilteredIndexCache struct { - cache IndexCache - enabledItems []string + cache IndexCache + postingsEnabled bool + seriesEnabled bool + expandedPostingsEnabled bool } // NewFilteredIndexCache creates a filtered index cache based on enabled items. func NewFilteredIndexCache(cache IndexCache, enabledItems []string) *FilteredIndexCache { - return &FilteredIndexCache{ - cache: cache, - enabledItems: enabledItems, - } + c := &FilteredIndexCache{cache: cache} + c.postingsEnabled = len(enabledItems) == 0 || slices.Contains(enabledItems, CacheTypePostings) + c.expandedPostingsEnabled = len(enabledItems) == 0 || slices.Contains(enabledItems, CacheTypeExpandedPostings) + c.seriesEnabled = len(enabledItems) == 0 || slices.Contains(enabledItems, CacheTypeSeries) + return c } // StorePostings sets the postings identified by the ulid and label to the value v, // if the postings already exists in the cache it is not mutated. func (c *FilteredIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte, tenant string) { - if len(c.enabledItems) == 0 || slices.Contains(c.enabledItems, CacheTypePostings) { + if c.postingsEnabled { c.cache.StorePostings(blockID, l, v, tenant) } } @@ -37,7 +40,7 @@ func (c *FilteredIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v // FetchMultiPostings fetches multiple postings - each identified by a label - // and returns a map containing cache hits, along with a list of missing keys. func (c *FilteredIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label, tenant string) (hits map[labels.Label][]byte, misses []labels.Label) { - if len(c.enabledItems) == 0 || slices.Contains(c.enabledItems, CacheTypePostings) { + if c.postingsEnabled { return c.cache.FetchMultiPostings(ctx, blockID, keys, tenant) } return nil, keys @@ -45,14 +48,14 @@ func (c *FilteredIndexCache) FetchMultiPostings(ctx context.Context, blockID uli // StoreExpandedPostings stores expanded postings for a set of label matchers. func (c *FilteredIndexCache) StoreExpandedPostings(blockID ulid.ULID, matchers []*labels.Matcher, v []byte, tenant string) { - if len(c.enabledItems) == 0 || slices.Contains(c.enabledItems, CacheTypeExpandedPostings) { + if c.expandedPostingsEnabled { c.cache.StoreExpandedPostings(blockID, matchers, v, tenant) } } // FetchExpandedPostings fetches expanded postings and returns cached data and a boolean value representing whether it is a cache hit or not. func (c *FilteredIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, matchers []*labels.Matcher, tenant string) ([]byte, bool) { - if len(c.enabledItems) == 0 || slices.Contains(c.enabledItems, CacheTypeExpandedPostings) { + if c.expandedPostingsEnabled { return c.cache.FetchExpandedPostings(ctx, blockID, matchers, tenant) } return nil, false @@ -61,7 +64,7 @@ func (c *FilteredIndexCache) FetchExpandedPostings(ctx context.Context, blockID // StoreSeries sets the series identified by the ulid and id to the value v, // if the series already exists in the cache it is not mutated. func (c *FilteredIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef, v []byte, tenant string) { - if len(c.enabledItems) == 0 || slices.Contains(c.enabledItems, CacheTypeSeries) { + if c.seriesEnabled { c.cache.StoreSeries(blockID, id, v, tenant) } } @@ -69,7 +72,7 @@ func (c *FilteredIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef // FetchMultiSeries fetches multiple series - each identified by ID - from the cache // and returns a map containing cache hits, along with a list of missing IDs. func (c *FilteredIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef, tenant string) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef) { - if len(c.enabledItems) == 0 || slices.Contains(c.enabledItems, CacheTypeSeries) { + if c.seriesEnabled { return c.cache.FetchMultiSeries(ctx, blockID, ids, tenant) } return nil, ids diff --git a/vendor/go.opentelemetry.io/collector/featuregate/LICENSE b/vendor/go.opentelemetry.io/collector/featuregate/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/go.opentelemetry.io/collector/featuregate/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/go.opentelemetry.io/collector/featuregate/Makefile b/vendor/go.opentelemetry.io/collector/featuregate/Makefile new file mode 100644 index 0000000000..39734bfaeb --- /dev/null +++ b/vendor/go.opentelemetry.io/collector/featuregate/Makefile @@ -0,0 +1 @@ +include ../Makefile.Common diff --git a/vendor/go.opentelemetry.io/collector/featuregate/README.md b/vendor/go.opentelemetry.io/collector/featuregate/README.md new file mode 100644 index 0000000000..fe7c4044b8 --- /dev/null +++ b/vendor/go.opentelemetry.io/collector/featuregate/README.md @@ -0,0 +1,73 @@ +# Collector Feature Gates + +This package provides a mechanism that allows operators to enable and disable +experimental or transitional features at deployment time. These flags should +be able to govern the behavior of the application starting as early as possible +and should be available to every component such that decisions may be made +based on flags at the component level. + +## Usage + +Feature gates must be defined and registered with the global registry in +an `init()` function. This makes the `Gate` available to be configured and +queried with the defined [`Stage`](#feature-lifecycle) default value. +A `Gate` can have a list of associated issues that allow users to refer to +the issue and report any additional problems or understand the context of the `Gate`. +Once a `Gate` has been marked as `Stable`, it must have a `RemovalVersion` set. + +```go +var myFeatureGate = featuregate.GlobalRegistry().MustRegister( + "namespaced.uniqueIdentifier", + featuregate.Stable, + featuregate.WithRegisterFromVersion("v0.65.0") + featuregate.WithRegisterDescription("A brief description of what the gate controls"), + featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector/issues/6167"), + featuregate.WithRegisterToVersion("v0.70.0")) +``` + +The status of the gate may later be checked by interrogating the global +feature gate registry: + +```go +if myFeatureGate.IsEnabled() { + setupNewFeature() +} +``` + +Note that querying the registry takes a read lock and accesses a map, so it +should be done once and the result cached for local use if repeated checks +are required. Avoid querying the registry in a loop. + +## Controlling Gates + +Feature gates can be enabled or disabled via the CLI, with the +`--feature-gates` flag. When using the CLI flag, gate +identifiers must be presented as a comma-delimited list. Gate identifiers +prefixed with `-` will disable the gate and prefixing with `+` or with no +prefix will enable the gate. + +```shell +otelcol --config=config.yaml --feature-gates=gate1,-gate2,+gate3 +``` + +This will enable `gate1` and `gate3` and disable `gate2`. + +## Feature Lifecycle + +Features controlled by a `Gate` should follow a three-stage lifecycle, +modeled after the [system used by Kubernetes](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-stages): + +1. An `alpha` stage where the feature is disabled by default and must be enabled + through a `Gate`. +2. A `beta` stage where the feature has been well tested and is enabled by + default but can be disabled through a `Gate`. +3. A generally available or `stable` stage where the feature is permanently enabled. At this stage + the gate should no longer be explicitly used. Disabling the gate will produce an error and + explicitly enabling will produce a warning log. +4. A `stable` feature gate will be removed in the version specified by its `ToVersion` value. + +Features that prove unworkable in the `alpha` stage may be discontinued +without proceeding to the `beta` stage. Features that make it to the `beta` +stage will not be dropped and will eventually reach general availability +where the `Gate` that allowed them to be disabled during the `beta` stage +will be removed. diff --git a/vendor/go.opentelemetry.io/collector/featuregate/flag.go b/vendor/go.opentelemetry.io/collector/featuregate/flag.go new file mode 100644 index 0000000000..64d6ec0c5c --- /dev/null +++ b/vendor/go.opentelemetry.io/collector/featuregate/flag.go @@ -0,0 +1,66 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package featuregate // import "go.opentelemetry.io/collector/featuregate" + +import ( + "flag" + "strings" + + "go.uber.org/multierr" +) + +// NewFlag returns a flag.Value that directly applies feature gate statuses to a Registry. +func NewFlag(reg *Registry) flag.Value { + return &flagValue{reg: reg} +} + +// flagValue implements the flag.Value interface and directly applies feature gate statuses to a Registry. +type flagValue struct { + reg *Registry +} + +func (f *flagValue) String() string { + var ids []string + f.reg.VisitAll(func(g *Gate) { + id := g.ID() + if !g.IsEnabled() { + id = "-" + id + } + ids = append(ids, id) + }) + return strings.Join(ids, ",") +} + +func (f *flagValue) Set(s string) error { + if s == "" { + return nil + } + + var errs error + ids := strings.Split(s, ",") + for i := range ids { + id := ids[i] + val := true + switch id[0] { + case '-': + id = id[1:] + val = false + case '+': + id = id[1:] + } + errs = multierr.Append(errs, f.reg.Set(id, val)) + } + return errs +} diff --git a/vendor/go.opentelemetry.io/collector/featuregate/gate.go b/vendor/go.opentelemetry.io/collector/featuregate/gate.go new file mode 100644 index 0000000000..2ba4ec7899 --- /dev/null +++ b/vendor/go.opentelemetry.io/collector/featuregate/gate.go @@ -0,0 +1,64 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package featuregate // import "go.opentelemetry.io/collector/featuregate" + +import "sync/atomic" + +// Gate is an immutable object that is owned by the Registry and represents an individual feature that +// may be enabled or disabled based on the lifecycle state of the feature and CLI flags specified by the user. +type Gate struct { + id string + description string + referenceURL string + fromVersion string + toVersion string + stage Stage + enabled *atomic.Bool +} + +// ID returns the id of the Gate. +func (g *Gate) ID() string { + return g.id +} + +// IsEnabled returns true if the feature described by the Gate is enabled. +func (g *Gate) IsEnabled() bool { + return g.enabled.Load() +} + +// Description returns the description for the Gate. +func (g *Gate) Description() string { + return g.description +} + +// Stage returns the Gate's lifecycle stage. +func (g *Gate) Stage() Stage { + return g.stage +} + +// ReferenceURL returns the URL to the contextual information about the Gate. +func (g *Gate) ReferenceURL() string { + return g.referenceURL +} + +// FromVersion returns the version information when the Gate's was added. +func (g *Gate) FromVersion() string { + return g.fromVersion +} + +// ToVersion returns the version information when Gate's in StageStable. +func (g *Gate) ToVersion() string { + return g.toVersion +} diff --git a/vendor/go.opentelemetry.io/collector/featuregate/registry.go b/vendor/go.opentelemetry.io/collector/featuregate/registry.go new file mode 100644 index 0000000000..0474f0c205 --- /dev/null +++ b/vendor/go.opentelemetry.io/collector/featuregate/registry.go @@ -0,0 +1,149 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package featuregate // import "go.opentelemetry.io/collector/featuregate" + +import ( + "fmt" + "sort" + "sync" + "sync/atomic" +) + +var globalRegistry = NewRegistry() + +// GlobalRegistry returns the global Registry. +func GlobalRegistry() *Registry { + return globalRegistry +} + +type Registry struct { + gates sync.Map +} + +// NewRegistry returns a new empty Registry. +func NewRegistry() *Registry { + return &Registry{} +} + +// RegisterOption allows to configure additional information about a Gate during registration. +type RegisterOption interface { + apply(g *Gate) +} + +type registerOptionFunc func(g *Gate) + +func (ro registerOptionFunc) apply(g *Gate) { + ro(g) +} + +// WithRegisterDescription adds description for the Gate. +func WithRegisterDescription(description string) RegisterOption { + return registerOptionFunc(func(g *Gate) { + g.description = description + }) +} + +// WithRegisterReferenceURL adds a URL that has all the contextual information about the Gate. +func WithRegisterReferenceURL(url string) RegisterOption { + return registerOptionFunc(func(g *Gate) { + g.referenceURL = url + }) +} + +// WithRegisterFromVersion is used to set the Gate "FromVersion". +// The "FromVersion" contains the Collector release when a feature is introduced. +func WithRegisterFromVersion(fromVersion string) RegisterOption { + return registerOptionFunc(func(g *Gate) { + g.fromVersion = fromVersion + }) +} + +// WithRegisterToVersion is used to set the Gate "ToVersion". +// The "ToVersion", if not empty, contains the last Collector release in which you can still use a feature gate. +// If the feature stage is either "Deprecated" or "Stable", the "ToVersion" is the Collector release when the feature is removed. +func WithRegisterToVersion(toVersion string) RegisterOption { + return registerOptionFunc(func(g *Gate) { + g.toVersion = toVersion + }) +} + +// MustRegister like Register but panics if an invalid ID or gate options are provided. +func (r *Registry) MustRegister(id string, stage Stage, opts ...RegisterOption) *Gate { + g, err := r.Register(id, stage, opts...) + if err != nil { + panic(err) + } + return g +} + +// Register a Gate and return it. The returned Gate can be used to check if is enabled or not. +func (r *Registry) Register(id string, stage Stage, opts ...RegisterOption) (*Gate, error) { + g := &Gate{ + id: id, + stage: stage, + } + for _, opt := range opts { + opt.apply(g) + } + switch g.stage { + case StageAlpha: + g.enabled = &atomic.Bool{} + case StageBeta, StageStable: + enabled := &atomic.Bool{} + enabled.Store(true) + g.enabled = enabled + default: + return nil, fmt.Errorf("unknown stage value %q for gate %q", stage, id) + } + if g.stage == StageStable && g.toVersion == "" { + return nil, fmt.Errorf("no removal version set for stable gate %q", id) + } + if _, loaded := r.gates.LoadOrStore(id, g); loaded { + return nil, fmt.Errorf("attempted to add pre-existing gate %q", id) + } + return g, nil +} + +// Set the enabled valued for a Gate identified by the given id. +func (r *Registry) Set(id string, enabled bool) error { + v, ok := r.gates.Load(id) + if !ok { + return fmt.Errorf("no such feature gate %q", id) + } + g := v.(*Gate) + if g.stage == StageStable { + if !enabled { + return fmt.Errorf("feature gate %q is stable, can not be disabled", id) + } + fmt.Printf("Feature gate %q is stable and already enabled. It will be removed in version %v and continued use of the gate after version %v will result in an error.\n", id, g.toVersion, g.toVersion) + } + g.enabled.Store(enabled) + return nil +} + +// VisitAll visits all the gates in lexicographical order, calling fn for each. +func (r *Registry) VisitAll(fn func(*Gate)) { + var gates []*Gate + r.gates.Range(func(key, value any) bool { + gates = append(gates, value.(*Gate)) + return true + }) + sort.Slice(gates, func(i, j int) bool { + return gates[i].ID() < gates[j].ID() + }) + for i := range gates { + fn(gates[i]) + } +} diff --git a/vendor/go.opentelemetry.io/collector/featuregate/stage.go b/vendor/go.opentelemetry.io/collector/featuregate/stage.go new file mode 100644 index 0000000000..2034e1c80d --- /dev/null +++ b/vendor/go.opentelemetry.io/collector/featuregate/stage.go @@ -0,0 +1,55 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package featuregate // import "go.opentelemetry.io/collector/featuregate" + +// Stage represents the Gate's lifecycle and what is the expected state of it. +type Stage int8 + +const ( + // StageAlpha is used when creating a new feature and the Gate must be explicitly enabled + // by the operator. + // + // The Gate will be disabled by default. + StageAlpha Stage = iota + // StageBeta is used when the feature gate is well tested and is enabled by default, + // but can be disabled by a Gate. + // + // The Gate will be enabled by default. + StageBeta + // StageStable is used when feature is permanently enabled and can not be disabled by a Gate. + // This value is used to provide feedback to the user that the gate will be removed in the next versions. + // + // The Gate will be enabled by default and will return an error if disabled. + StageStable + // StageDeprecated is used when feature is permanently disabled and can not be enabled by a Gate. + // This value is used to provide feedback to the user that the gate will be removed in the next versions. + // + // The Gate will be disabled by default and will return an error if modified. + StageDeprecated +) + +func (s Stage) String() string { + switch s { + case StageAlpha: + return "Alpha" + case StageBeta: + return "Beta" + case StageStable: + return "Stable" + case StageDeprecated: + return "Deprecated" + } + return "Unknown" +} diff --git a/vendor/go.opentelemetry.io/collector/pdata/pmetric/metrics.go b/vendor/go.opentelemetry.io/collector/pdata/pmetric/metrics.go index 5a8c0f2997..91195ca4df 100644 --- a/vendor/go.opentelemetry.io/collector/pdata/pmetric/metrics.go +++ b/vendor/go.opentelemetry.io/collector/pdata/pmetric/metrics.go @@ -21,11 +21,20 @@ func (ms Metrics) getOrig() *otlpcollectormetrics.ExportMetricsServiceRequest { return internal.GetOrigMetrics(internal.Metrics(ms)) } +func (ms Metrics) getState() *internal.State { + return internal.GetMetricsState(internal.Metrics(ms)) +} + // NewMetrics creates a new Metrics struct. func NewMetrics() Metrics { return newMetrics(&otlpcollectormetrics.ExportMetricsServiceRequest{}) } +// IsReadOnly returns true if this Metrics instance is read-only. +func (ms Metrics) IsReadOnly() bool { + return *ms.getState() == internal.StateReadOnly +} + // CopyTo copies the Metrics instance overriding the destination. func (ms Metrics) CopyTo(dest Metrics) { ms.ResourceMetrics().CopyTo(dest.ResourceMetrics()) diff --git a/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go b/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go index 0454cdd78e..333676b7cf 100644 --- a/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go +++ b/vendor/golang.org/x/tools/go/internal/packagesdriver/sizes.go @@ -13,16 +13,17 @@ import ( "golang.org/x/tools/internal/gocommand" ) -var debug = false - func GetSizesForArgsGolist(ctx context.Context, inv gocommand.Invocation, gocmdRunner *gocommand.Runner) (string, string, error) { inv.Verb = "list" inv.Args = []string{"-f", "{{context.GOARCH}} {{context.Compiler}}", "--", "unsafe"} stdout, stderr, friendlyErr, rawErr := gocmdRunner.RunRaw(ctx, inv) var goarch, compiler string if rawErr != nil { - if rawErrMsg := rawErr.Error(); strings.Contains(rawErrMsg, "cannot find main module") || strings.Contains(rawErrMsg, "go.mod file not found") { - // User's running outside of a module. All bets are off. Get GOARCH and guess compiler is gc. + rawErrMsg := rawErr.Error() + if strings.Contains(rawErrMsg, "cannot find main module") || + strings.Contains(rawErrMsg, "go.mod file not found") { + // User's running outside of a module. + // All bets are off. Get GOARCH and guess compiler is gc. // TODO(matloob): Is this a problem in practice? inv.Verb = "env" inv.Args = []string{"GOARCH"} @@ -32,8 +33,12 @@ func GetSizesForArgsGolist(ctx context.Context, inv gocommand.Invocation, gocmdR } goarch = strings.TrimSpace(envout.String()) compiler = "gc" - } else { + } else if friendlyErr != nil { return "", "", friendlyErr + } else { + // This should be unreachable, but be defensive + // in case RunRaw's error results are inconsistent. + return "", "", rawErr } } else { fields := strings.Fields(stdout.String()) diff --git a/vendor/golang.org/x/tools/go/packages/golist.go b/vendor/golang.org/x/tools/go/packages/golist.go index 1f1eade0ac..c1292b30f3 100644 --- a/vendor/golang.org/x/tools/go/packages/golist.go +++ b/vendor/golang.org/x/tools/go/packages/golist.go @@ -208,62 +208,6 @@ extractQueries: } } - // Only use go/packages' overlay processing if we're using a Go version - // below 1.16. Otherwise, go list handles it. - if goVersion, err := state.getGoVersion(); err == nil && goVersion < 16 { - modifiedPkgs, needPkgs, err := state.processGolistOverlay(response) - if err != nil { - return nil, err - } - - var containsCandidates []string - if len(containFiles) > 0 { - containsCandidates = append(containsCandidates, modifiedPkgs...) - containsCandidates = append(containsCandidates, needPkgs...) - } - if err := state.addNeededOverlayPackages(response, needPkgs); err != nil { - return nil, err - } - // Check candidate packages for containFiles. - if len(containFiles) > 0 { - for _, id := range containsCandidates { - pkg, ok := response.seenPackages[id] - if !ok { - response.addPackage(&Package{ - ID: id, - Errors: []Error{{ - Kind: ListError, - Msg: fmt.Sprintf("package %s expected but not seen", id), - }}, - }) - continue - } - for _, f := range containFiles { - for _, g := range pkg.GoFiles { - if sameFile(f, g) { - response.addRoot(id) - } - } - } - } - } - // Add root for any package that matches a pattern. This applies only to - // packages that are modified by overlays, since they are not added as - // roots automatically. - for _, pattern := range restPatterns { - match := matchPattern(pattern) - for _, pkgID := range modifiedPkgs { - pkg, ok := response.seenPackages[pkgID] - if !ok { - continue - } - if match(pkg.PkgPath) { - response.addRoot(pkg.ID) - } - } - } - } - sizeswg.Wait() if sizeserr != nil { return nil, sizeserr @@ -271,24 +215,6 @@ extractQueries: return response.dr, nil } -func (state *golistState) addNeededOverlayPackages(response *responseDeduper, pkgs []string) error { - if len(pkgs) == 0 { - return nil - } - dr, err := state.createDriverResponse(pkgs...) - if err != nil { - return err - } - for _, pkg := range dr.Packages { - response.addPackage(pkg) - } - _, needPkgs, err := state.processGolistOverlay(response) - if err != nil { - return err - } - return state.addNeededOverlayPackages(response, needPkgs) -} - func (state *golistState) runContainsQueries(response *responseDeduper, queries []string) error { for _, query := range queries { // TODO(matloob): Do only one query per directory. diff --git a/vendor/golang.org/x/tools/go/packages/golist_overlay.go b/vendor/golang.org/x/tools/go/packages/golist_overlay.go index 9576b472f9..d823c474ad 100644 --- a/vendor/golang.org/x/tools/go/packages/golist_overlay.go +++ b/vendor/golang.org/x/tools/go/packages/golist_overlay.go @@ -6,314 +6,11 @@ package packages import ( "encoding/json" - "fmt" - "go/parser" - "go/token" - "os" "path/filepath" - "regexp" - "sort" - "strconv" - "strings" "golang.org/x/tools/internal/gocommand" ) -// processGolistOverlay provides rudimentary support for adding -// files that don't exist on disk to an overlay. The results can be -// sometimes incorrect. -// TODO(matloob): Handle unsupported cases, including the following: -// - determining the correct package to add given a new import path -func (state *golistState) processGolistOverlay(response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) { - havePkgs := make(map[string]string) // importPath -> non-test package ID - needPkgsSet := make(map[string]bool) - modifiedPkgsSet := make(map[string]bool) - - pkgOfDir := make(map[string][]*Package) - for _, pkg := range response.dr.Packages { - // This is an approximation of import path to id. This can be - // wrong for tests, vendored packages, and a number of other cases. - havePkgs[pkg.PkgPath] = pkg.ID - dir, err := commonDir(pkg.GoFiles) - if err != nil { - return nil, nil, err - } - if dir != "" { - pkgOfDir[dir] = append(pkgOfDir[dir], pkg) - } - } - - // If no new imports are added, it is safe to avoid loading any needPkgs. - // Otherwise, it's hard to tell which package is actually being loaded - // (due to vendoring) and whether any modified package will show up - // in the transitive set of dependencies (because new imports are added, - // potentially modifying the transitive set of dependencies). - var overlayAddsImports bool - - // If both a package and its test package are created by the overlay, we - // need the real package first. Process all non-test files before test - // files, and make the whole process deterministic while we're at it. - var overlayFiles []string - for opath := range state.cfg.Overlay { - overlayFiles = append(overlayFiles, opath) - } - sort.Slice(overlayFiles, func(i, j int) bool { - iTest := strings.HasSuffix(overlayFiles[i], "_test.go") - jTest := strings.HasSuffix(overlayFiles[j], "_test.go") - if iTest != jTest { - return !iTest // non-tests are before tests. - } - return overlayFiles[i] < overlayFiles[j] - }) - for _, opath := range overlayFiles { - contents := state.cfg.Overlay[opath] - base := filepath.Base(opath) - dir := filepath.Dir(opath) - var pkg *Package // if opath belongs to both a package and its test variant, this will be the test variant - var testVariantOf *Package // if opath is a test file, this is the package it is testing - var fileExists bool - isTestFile := strings.HasSuffix(opath, "_test.go") - pkgName, ok := extractPackageName(opath, contents) - if !ok { - // Don't bother adding a file that doesn't even have a parsable package statement - // to the overlay. - continue - } - // If all the overlay files belong to a different package, change the - // package name to that package. - maybeFixPackageName(pkgName, isTestFile, pkgOfDir[dir]) - nextPackage: - for _, p := range response.dr.Packages { - if pkgName != p.Name && p.ID != "command-line-arguments" { - continue - } - for _, f := range p.GoFiles { - if !sameFile(filepath.Dir(f), dir) { - continue - } - // Make sure to capture information on the package's test variant, if needed. - if isTestFile && !hasTestFiles(p) { - // TODO(matloob): Are there packages other than the 'production' variant - // of a package that this can match? This shouldn't match the test main package - // because the file is generated in another directory. - testVariantOf = p - continue nextPackage - } else if !isTestFile && hasTestFiles(p) { - // We're examining a test variant, but the overlaid file is - // a non-test file. Because the overlay implementation - // (currently) only adds a file to one package, skip this - // package, so that we can add the file to the production - // variant of the package. (https://golang.org/issue/36857 - // tracks handling overlays on both the production and test - // variant of a package). - continue nextPackage - } - if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath { - // We have already seen the production version of the - // for which p is a test variant. - if hasTestFiles(p) { - testVariantOf = pkg - } - } - pkg = p - if filepath.Base(f) == base { - fileExists = true - } - } - } - // The overlay could have included an entirely new package or an - // ad-hoc package. An ad-hoc package is one that we have manually - // constructed from inadequate `go list` results for a file= query. - // It will have the ID command-line-arguments. - if pkg == nil || pkg.ID == "command-line-arguments" { - // Try to find the module or gopath dir the file is contained in. - // Then for modules, add the module opath to the beginning. - pkgPath, ok, err := state.getPkgPath(dir) - if err != nil { - return nil, nil, err - } - if !ok { - break - } - var forTest string // only set for x tests - isXTest := strings.HasSuffix(pkgName, "_test") - if isXTest { - forTest = pkgPath - pkgPath += "_test" - } - id := pkgPath - if isTestFile { - if isXTest { - id = fmt.Sprintf("%s [%s.test]", pkgPath, forTest) - } else { - id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath) - } - } - if pkg != nil { - // TODO(rstambler): We should change the package's path and ID - // here. The only issue is that this messes with the roots. - } else { - // Try to reclaim a package with the same ID, if it exists in the response. - for _, p := range response.dr.Packages { - if reclaimPackage(p, id, opath, contents) { - pkg = p - break - } - } - // Otherwise, create a new package. - if pkg == nil { - pkg = &Package{ - PkgPath: pkgPath, - ID: id, - Name: pkgName, - Imports: make(map[string]*Package), - } - response.addPackage(pkg) - havePkgs[pkg.PkgPath] = id - // Add the production package's sources for a test variant. - if isTestFile && !isXTest && testVariantOf != nil { - pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...) - pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...) - // Add the package under test and its imports to the test variant. - pkg.forTest = testVariantOf.PkgPath - for k, v := range testVariantOf.Imports { - pkg.Imports[k] = &Package{ID: v.ID} - } - } - if isXTest { - pkg.forTest = forTest - } - } - } - } - if !fileExists { - pkg.GoFiles = append(pkg.GoFiles, opath) - // TODO(matloob): Adding the file to CompiledGoFiles can exhibit the wrong behavior - // if the file will be ignored due to its build tags. - pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, opath) - modifiedPkgsSet[pkg.ID] = true - } - imports, err := extractImports(opath, contents) - if err != nil { - // Let the parser or type checker report errors later. - continue - } - for _, imp := range imports { - // TODO(rstambler): If the package is an x test and the import has - // a test variant, make sure to replace it. - if _, found := pkg.Imports[imp]; found { - continue - } - overlayAddsImports = true - id, ok := havePkgs[imp] - if !ok { - var err error - id, err = state.resolveImport(dir, imp) - if err != nil { - return nil, nil, err - } - } - pkg.Imports[imp] = &Package{ID: id} - // Add dependencies to the non-test variant version of this package as well. - if testVariantOf != nil { - testVariantOf.Imports[imp] = &Package{ID: id} - } - } - } - - // toPkgPath guesses the package path given the id. - toPkgPath := func(sourceDir, id string) (string, error) { - if i := strings.IndexByte(id, ' '); i >= 0 { - return state.resolveImport(sourceDir, id[:i]) - } - return state.resolveImport(sourceDir, id) - } - - // Now that new packages have been created, do another pass to determine - // the new set of missing packages. - for _, pkg := range response.dr.Packages { - for _, imp := range pkg.Imports { - if len(pkg.GoFiles) == 0 { - return nil, nil, fmt.Errorf("cannot resolve imports for package %q with no Go files", pkg.PkgPath) - } - pkgPath, err := toPkgPath(filepath.Dir(pkg.GoFiles[0]), imp.ID) - if err != nil { - return nil, nil, err - } - if _, ok := havePkgs[pkgPath]; !ok { - needPkgsSet[pkgPath] = true - } - } - } - - if overlayAddsImports { - needPkgs = make([]string, 0, len(needPkgsSet)) - for pkg := range needPkgsSet { - needPkgs = append(needPkgs, pkg) - } - } - modifiedPkgs = make([]string, 0, len(modifiedPkgsSet)) - for pkg := range modifiedPkgsSet { - modifiedPkgs = append(modifiedPkgs, pkg) - } - return modifiedPkgs, needPkgs, err -} - -// resolveImport finds the ID of a package given its import path. -// In particular, it will find the right vendored copy when in GOPATH mode. -func (state *golistState) resolveImport(sourceDir, importPath string) (string, error) { - env, err := state.getEnv() - if err != nil { - return "", err - } - if env["GOMOD"] != "" { - return importPath, nil - } - - searchDir := sourceDir - for { - vendorDir := filepath.Join(searchDir, "vendor") - exists, ok := state.vendorDirs[vendorDir] - if !ok { - info, err := os.Stat(vendorDir) - exists = err == nil && info.IsDir() - state.vendorDirs[vendorDir] = exists - } - - if exists { - vendoredPath := filepath.Join(vendorDir, importPath) - if info, err := os.Stat(vendoredPath); err == nil && info.IsDir() { - // We should probably check for .go files here, but shame on anyone who fools us. - path, ok, err := state.getPkgPath(vendoredPath) - if err != nil { - return "", err - } - if ok { - return path, nil - } - } - } - - // We know we've hit the top of the filesystem when we Dir / and get /, - // or C:\ and get C:\, etc. - next := filepath.Dir(searchDir) - if next == searchDir { - break - } - searchDir = next - } - return importPath, nil -} - -func hasTestFiles(p *Package) bool { - for _, f := range p.GoFiles { - if strings.HasSuffix(f, "_test.go") { - return true - } - } - return false -} - // determineRootDirs returns a mapping from absolute directories that could // contain code to their corresponding import path prefixes. func (state *golistState) determineRootDirs() (map[string]string, error) { @@ -384,192 +81,3 @@ func (state *golistState) determineRootDirsGOPATH() (map[string]string, error) { } return m, nil } - -func extractImports(filename string, contents []byte) ([]string, error) { - f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset? - if err != nil { - return nil, err - } - var res []string - for _, imp := range f.Imports { - quotedPath := imp.Path.Value - path, err := strconv.Unquote(quotedPath) - if err != nil { - return nil, err - } - res = append(res, path) - } - return res, nil -} - -// reclaimPackage attempts to reuse a package that failed to load in an overlay. -// -// If the package has errors and has no Name, GoFiles, or Imports, -// then it's possible that it doesn't yet exist on disk. -func reclaimPackage(pkg *Package, id string, filename string, contents []byte) bool { - // TODO(rstambler): Check the message of the actual error? - // It differs between $GOPATH and module mode. - if pkg.ID != id { - return false - } - if len(pkg.Errors) != 1 { - return false - } - if pkg.Name != "" || pkg.ExportFile != "" { - return false - } - if len(pkg.GoFiles) > 0 || len(pkg.CompiledGoFiles) > 0 || len(pkg.OtherFiles) > 0 { - return false - } - if len(pkg.Imports) > 0 { - return false - } - pkgName, ok := extractPackageName(filename, contents) - if !ok { - return false - } - pkg.Name = pkgName - pkg.Errors = nil - return true -} - -func extractPackageName(filename string, contents []byte) (string, bool) { - // TODO(rstambler): Check the message of the actual error? - // It differs between $GOPATH and module mode. - f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.PackageClauseOnly) // TODO(matloob): reuse fileset? - if err != nil { - return "", false - } - return f.Name.Name, true -} - -// commonDir returns the directory that all files are in, "" if files is empty, -// or an error if they aren't in the same directory. -func commonDir(files []string) (string, error) { - seen := make(map[string]bool) - for _, f := range files { - seen[filepath.Dir(f)] = true - } - if len(seen) > 1 { - return "", fmt.Errorf("files (%v) are in more than one directory: %v", files, seen) - } - for k := range seen { - // seen has only one element; return it. - return k, nil - } - return "", nil // no files -} - -// It is possible that the files in the disk directory dir have a different package -// name from newName, which is deduced from the overlays. If they all have a different -// package name, and they all have the same package name, then that name becomes -// the package name. -// It returns true if it changes the package name, false otherwise. -func maybeFixPackageName(newName string, isTestFile bool, pkgsOfDir []*Package) { - names := make(map[string]int) - for _, p := range pkgsOfDir { - names[p.Name]++ - } - if len(names) != 1 { - // some files are in different packages - return - } - var oldName string - for k := range names { - oldName = k - } - if newName == oldName { - return - } - // We might have a case where all of the package names in the directory are - // the same, but the overlay file is for an x test, which belongs to its - // own package. If the x test does not yet exist on disk, we may not yet - // have its package name on disk, but we should not rename the packages. - // - // We use a heuristic to determine if this file belongs to an x test: - // The test file should have a package name whose package name has a _test - // suffix or looks like "newName_test". - maybeXTest := strings.HasPrefix(oldName+"_test", newName) || strings.HasSuffix(newName, "_test") - if isTestFile && maybeXTest { - return - } - for _, p := range pkgsOfDir { - p.Name = newName - } -} - -// This function is copy-pasted from -// https://github.com/golang/go/blob/9706f510a5e2754595d716bd64be8375997311fb/src/cmd/go/internal/search/search.go#L360. -// It should be deleted when we remove support for overlays from go/packages. -// -// NOTE: This does not handle any ./... or ./ style queries, as this function -// doesn't know the working directory. -// -// matchPattern(pattern)(name) reports whether -// name matches pattern. Pattern is a limited glob -// pattern in which '...' means 'any string' and there -// is no other special syntax. -// Unfortunately, there are two special cases. Quoting "go help packages": -// -// First, /... at the end of the pattern can match an empty string, -// so that net/... matches both net and packages in its subdirectories, like net/http. -// Second, any slash-separated pattern element containing a wildcard never -// participates in a match of the "vendor" element in the path of a vendored -// package, so that ./... does not match packages in subdirectories of -// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. -// Note, however, that a directory named vendor that itself contains code -// is not a vendored package: cmd/vendor would be a command named vendor, -// and the pattern cmd/... matches it. -func matchPattern(pattern string) func(name string) bool { - // Convert pattern to regular expression. - // The strategy for the trailing /... is to nest it in an explicit ? expression. - // The strategy for the vendor exclusion is to change the unmatchable - // vendor strings to a disallowed code point (vendorChar) and to use - // "(anything but that codepoint)*" as the implementation of the ... wildcard. - // This is a bit complicated but the obvious alternative, - // namely a hand-written search like in most shell glob matchers, - // is too easy to make accidentally exponential. - // Using package regexp guarantees linear-time matching. - - const vendorChar = "\x00" - - if strings.Contains(pattern, vendorChar) { - return func(name string) bool { return false } - } - - re := regexp.QuoteMeta(pattern) - re = replaceVendor(re, vendorChar) - switch { - case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`): - re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)` - case re == vendorChar+`/\.\.\.`: - re = `(/vendor|/` + vendorChar + `/\.\.\.)` - case strings.HasSuffix(re, `/\.\.\.`): - re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` - } - re = strings.ReplaceAll(re, `\.\.\.`, `[^`+vendorChar+`]*`) - - reg := regexp.MustCompile(`^` + re + `$`) - - return func(name string) bool { - if strings.Contains(name, vendorChar) { - return false - } - return reg.MatchString(replaceVendor(name, vendorChar)) - } -} - -// replaceVendor returns the result of replacing -// non-trailing vendor path elements in x with repl. -func replaceVendor(x, repl string) string { - if !strings.Contains(x, "vendor") { - return x - } - elem := strings.Split(x, "/") - for i := 0; i < len(elem)-1; i++ { - if elem[i] == "vendor" { - elem[i] = repl - } - } - return strings.Join(elem, "/") -} diff --git a/vendor/golang.org/x/tools/go/packages/packages.go b/vendor/golang.org/x/tools/go/packages/packages.go index ece0e7c603..6cbd3de83e 100644 --- a/vendor/golang.org/x/tools/go/packages/packages.go +++ b/vendor/golang.org/x/tools/go/packages/packages.go @@ -258,31 +258,52 @@ type driverResponse struct { // proceeding with further analysis. The PrintErrors function is // provided for convenient display of all errors. func Load(cfg *Config, patterns ...string) ([]*Package, error) { - l := newLoader(cfg) - response, err := defaultDriver(&l.Config, patterns...) + ld := newLoader(cfg) + response, external, err := defaultDriver(&ld.Config, patterns...) if err != nil { return nil, err } - l.sizes = types.SizesFor(response.Compiler, response.Arch) - return l.refine(response) + + ld.sizes = types.SizesFor(response.Compiler, response.Arch) + if ld.sizes == nil && ld.Config.Mode&(NeedTypes|NeedTypesSizes|NeedTypesInfo) != 0 { + // Type size information is needed but unavailable. + if external { + // An external driver may fail to populate the Compiler/GOARCH fields, + // especially since they are relatively new (see #63700). + // Provide a sensible fallback in this case. + ld.sizes = types.SizesFor("gc", runtime.GOARCH) + if ld.sizes == nil { // gccgo-only arch + ld.sizes = types.SizesFor("gc", "amd64") + } + } else { + // Go list should never fail to deliver accurate size information. + // Reject the whole Load since the error is the same for every package. + return nil, fmt.Errorf("can't determine type sizes for compiler %q on GOARCH %q", + response.Compiler, response.Arch) + } + } + + return ld.refine(response) } // defaultDriver is a driver that implements go/packages' fallback behavior. // It will try to request to an external driver, if one exists. If there's // no external driver, or the driver returns a response with NotHandled set, // defaultDriver will fall back to the go list driver. -func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) { - driver := findExternalDriver(cfg) - if driver == nil { - driver = goListDriver - } - response, err := driver(cfg, patterns...) - if err != nil { - return response, err - } else if response.NotHandled { - return goListDriver(cfg, patterns...) +// The boolean result indicates that an external driver handled the request. +func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, bool, error) { + if driver := findExternalDriver(cfg); driver != nil { + response, err := driver(cfg, patterns...) + if err != nil { + return nil, false, err + } else if !response.NotHandled { + return response, true, nil + } + // (fall through) } - return response, nil + + response, err := goListDriver(cfg, patterns...) + return response, false, err } // A Package describes a loaded Go package. @@ -553,7 +574,7 @@ type loaderPackage struct { type loader struct { pkgs map[string]*loaderPackage Config - sizes types.Sizes + sizes types.Sizes // non-nil if needed by mode parseCache map[string]*parseValue parseCacheMu sync.Mutex exportMu sync.Mutex // enforces mutual exclusion of exportdata operations @@ -678,39 +699,38 @@ func (ld *loader) refine(response *driverResponse) ([]*Package, error) { } } - // Materialize the import graph. - - const ( - white = 0 // new - grey = 1 // in progress - black = 2 // complete - ) - - // visit traverses the import graph, depth-first, - // and materializes the graph as Packages.Imports. - // - // Valid imports are saved in the Packages.Import map. - // Invalid imports (cycles and missing nodes) are saved in the importErrors map. - // Thus, even in the presence of both kinds of errors, the Import graph remains a DAG. - // - // visit returns whether the package needs src or has a transitive - // dependency on a package that does. These are the only packages - // for which we load source code. - var stack []*loaderPackage - var visit func(lpkg *loaderPackage) bool - var srcPkgs []*loaderPackage - visit = func(lpkg *loaderPackage) bool { - switch lpkg.color { - case black: - return lpkg.needsrc - case grey: - panic("internal error: grey node") - } - lpkg.color = grey - stack = append(stack, lpkg) // push - stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports - // If NeedImports isn't set, the imports fields will all be zeroed out. - if ld.Mode&NeedImports != 0 { + if ld.Mode&NeedImports != 0 { + // Materialize the import graph. + + const ( + white = 0 // new + grey = 1 // in progress + black = 2 // complete + ) + + // visit traverses the import graph, depth-first, + // and materializes the graph as Packages.Imports. + // + // Valid imports are saved in the Packages.Import map. + // Invalid imports (cycles and missing nodes) are saved in the importErrors map. + // Thus, even in the presence of both kinds of errors, + // the Import graph remains a DAG. + // + // visit returns whether the package needs src or has a transitive + // dependency on a package that does. These are the only packages + // for which we load source code. + var stack []*loaderPackage + var visit func(lpkg *loaderPackage) bool + visit = func(lpkg *loaderPackage) bool { + switch lpkg.color { + case black: + return lpkg.needsrc + case grey: + panic("internal error: grey node") + } + lpkg.color = grey + stack = append(stack, lpkg) // push + stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports lpkg.Imports = make(map[string]*Package, len(stubs)) for importPath, ipkg := range stubs { var importErr error @@ -734,40 +754,39 @@ func (ld *loader) refine(response *driverResponse) ([]*Package, error) { } lpkg.Imports[importPath] = imp.Package } - } - if lpkg.needsrc { - srcPkgs = append(srcPkgs, lpkg) - } - if ld.Mode&NeedTypesSizes != 0 { - lpkg.TypesSizes = ld.sizes - } - stack = stack[:len(stack)-1] // pop - lpkg.color = black - return lpkg.needsrc - } + // Complete type information is required for the + // immediate dependencies of each source package. + if lpkg.needsrc && ld.Mode&NeedTypes != 0 { + for _, ipkg := range lpkg.Imports { + ld.pkgs[ipkg.ID].needtypes = true + } + } - if ld.Mode&NeedImports == 0 { - // We do this to drop the stub import packages that we are not even going to try to resolve. - for _, lpkg := range initial { - lpkg.Imports = nil + // NeedTypeSizes causes TypeSizes to be set even + // on packages for which types aren't needed. + if ld.Mode&NeedTypesSizes != 0 { + lpkg.TypesSizes = ld.sizes + } + stack = stack[:len(stack)-1] // pop + lpkg.color = black + + return lpkg.needsrc } - } else { + // For each initial package, create its import DAG. for _, lpkg := range initial { visit(lpkg) } - } - if ld.Mode&NeedImports != 0 && ld.Mode&NeedTypes != 0 { - for _, lpkg := range srcPkgs { - // Complete type information is required for the - // immediate dependencies of each source package. - for _, ipkg := range lpkg.Imports { - imp := ld.pkgs[ipkg.ID] - imp.needtypes = true - } + + } else { + // !NeedImports: drop the stub (ID-only) import packages + // that we are not even going to try to resolve. + for _, lpkg := range initial { + lpkg.Imports = nil } } + // Load type data and syntax if needed, starting at // the initial packages (roots of the import DAG). if ld.Mode&NeedTypes != 0 || ld.Mode&NeedSyntax != 0 { @@ -1042,7 +1061,7 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) { IgnoreFuncBodies: ld.Mode&NeedDeps == 0 && !lpkg.initial, Error: appendError, - Sizes: ld.sizes, + Sizes: ld.sizes, // may be nil } if lpkg.Module != nil && lpkg.Module.GoVersion != "" { typesinternal.SetGoVersion(tc, "go"+lpkg.Module.GoVersion) diff --git a/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go b/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go index fa5834baf7..e742ecc464 100644 --- a/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go +++ b/vendor/golang.org/x/tools/go/types/objectpath/objectpath.go @@ -26,13 +26,10 @@ package objectpath import ( "fmt" "go/types" - "sort" "strconv" "strings" - _ "unsafe" "golang.org/x/tools/internal/typeparams" - "golang.org/x/tools/internal/typesinternal" ) // A Path is an opaque name that identifies a types.Object @@ -123,20 +120,7 @@ func For(obj types.Object) (Path, error) { // An Encoder amortizes the cost of encoding the paths of multiple objects. // The zero value of an Encoder is ready to use. type Encoder struct { - scopeMemo map[*types.Scope][]types.Object // memoization of scopeObjects - namedMethodsMemo map[*types.Named][]*types.Func // memoization of namedMethods() - skipMethodSorting bool -} - -// Expose back doors so that gopls can avoid method sorting, which can dominate -// analysis on certain repositories. -// -// TODO(golang/go#61443): remove this. -func init() { - typesinternal.SkipEncoderMethodSorting = func(enc interface{}) { - enc.(*Encoder).skipMethodSorting = true - } - typesinternal.ObjectpathObject = object + scopeMemo map[*types.Scope][]types.Object // memoization of scopeObjects } // For returns the path to an object relative to its package, @@ -328,31 +312,18 @@ func (enc *Encoder) For(obj types.Object) (Path, error) { // Inspect declared methods of defined types. if T, ok := o.Type().(*types.Named); ok { path = append(path, opType) - if !enc.skipMethodSorting { - // Note that method index here is always with respect - // to canonical ordering of methods, regardless of how - // they appear in the underlying type. - for i, m := range enc.namedMethods(T) { - path2 := appendOpArg(path, opMethod, i) - if m == obj { - return Path(path2), nil // found declared method - } - if r := find(obj, m.Type(), append(path2, opType), nil); r != nil { - return Path(r), nil - } + // The method index here is always with respect + // to the underlying go/types data structures, + // which ultimately derives from source order + // and must be preserved by export data. + for i := 0; i < T.NumMethods(); i++ { + m := T.Method(i) + path2 := appendOpArg(path, opMethod, i) + if m == obj { + return Path(path2), nil // found declared method } - } else { - // This branch must match the logic in the branch above, using go/types - // APIs without sorting. - for i := 0; i < T.NumMethods(); i++ { - m := T.Method(i) - path2 := appendOpArg(path, opMethod, i) - if m == obj { - return Path(path2), nil // found declared method - } - if r := find(obj, m.Type(), append(path2, opType), nil); r != nil { - return Path(r), nil - } + if r := find(obj, m.Type(), append(path2, opType), nil); r != nil { + return Path(r), nil } } } @@ -448,22 +419,13 @@ func (enc *Encoder) concreteMethod(meth *types.Func) (Path, bool) { path = append(path, name...) path = append(path, opType) - if !enc.skipMethodSorting { - for i, m := range enc.namedMethods(named) { - if m == meth { - path = appendOpArg(path, opMethod, i) - return Path(path), true - } - } - } else { - // This branch must match the logic of the branch above, using go/types - // APIs without sorting. - for i := 0; i < named.NumMethods(); i++ { - m := named.Method(i) - if m == meth { - path = appendOpArg(path, opMethod, i) - return Path(path), true - } + // Method indices are w.r.t. the go/types data structures, + // ultimately deriving from source order, + // which is preserved by export data. + for i := 0; i < named.NumMethods(); i++ { + if named.Method(i) == meth { + path = appendOpArg(path, opMethod, i) + return Path(path), true } } @@ -576,12 +538,7 @@ func findTypeParam(obj types.Object, list *typeparams.TypeParamList, path []byte // Object returns the object denoted by path p within the package pkg. func Object(pkg *types.Package, p Path) (types.Object, error) { - return object(pkg, string(p), false) -} - -// Note: the skipMethodSorting parameter must match the value of -// Encoder.skipMethodSorting used during encoding. -func object(pkg *types.Package, pathstr string, skipMethodSorting bool) (types.Object, error) { + pathstr := string(p) if pathstr == "" { return nil, fmt.Errorf("empty path") } @@ -747,12 +704,7 @@ func object(pkg *types.Package, pathstr string, skipMethodSorting bool) (types.O if index >= t.NumMethods() { return nil, fmt.Errorf("method index %d out of range [0-%d)", index, t.NumMethods()) } - if skipMethodSorting { - obj = t.Method(index) - } else { - methods := namedMethods(t) // (unmemoized) - obj = methods[index] // Id-ordered - } + obj = t.Method(index) default: return nil, fmt.Errorf("cannot apply %q to %s (got %T, want interface or named)", code, t, t) @@ -779,33 +731,6 @@ func object(pkg *types.Package, pathstr string, skipMethodSorting bool) (types.O return obj, nil // success } -// namedMethods returns the methods of a Named type in ascending Id order. -func namedMethods(named *types.Named) []*types.Func { - methods := make([]*types.Func, named.NumMethods()) - for i := range methods { - methods[i] = named.Method(i) - } - sort.Slice(methods, func(i, j int) bool { - return methods[i].Id() < methods[j].Id() - }) - return methods -} - -// namedMethods is a memoization of the namedMethods function. Callers must not modify the result. -func (enc *Encoder) namedMethods(named *types.Named) []*types.Func { - m := enc.namedMethodsMemo - if m == nil { - m = make(map[*types.Named][]*types.Func) - enc.namedMethodsMemo = m - } - methods, ok := m[named] - if !ok { - methods = namedMethods(named) // allocates and sorts - m[named] = methods - } - return methods -} - // scopeObjects is a memoization of scope objects. // Callers must not modify the result. func (enc *Encoder) scopeObjects(scope *types.Scope) []types.Object { diff --git a/vendor/golang.org/x/tools/internal/gocommand/invoke.go b/vendor/golang.org/x/tools/internal/gocommand/invoke.go index 53cf66da01..c27b91f8c7 100644 --- a/vendor/golang.org/x/tools/internal/gocommand/invoke.go +++ b/vendor/golang.org/x/tools/internal/gocommand/invoke.go @@ -85,6 +85,7 @@ func (runner *Runner) RunPiped(ctx context.Context, inv Invocation, stdout, stde // RunRaw runs the invocation, serializing requests only if they fight over // go.mod changes. +// Postcondition: both error results have same nilness. func (runner *Runner) RunRaw(ctx context.Context, inv Invocation) (*bytes.Buffer, *bytes.Buffer, error, error) { ctx, done := event.Start(ctx, "gocommand.Runner.RunRaw", invLabels(inv)...) defer done() @@ -95,23 +96,24 @@ func (runner *Runner) RunRaw(ctx context.Context, inv Invocation) (*bytes.Buffer stdout, stderr, friendlyErr, err := runner.runConcurrent(ctx, inv) // If we encounter a load concurrency error, we need to retry serially. - if friendlyErr == nil || !modConcurrencyError.MatchString(friendlyErr.Error()) { - return stdout, stderr, friendlyErr, err + if friendlyErr != nil && modConcurrencyError.MatchString(friendlyErr.Error()) { + event.Error(ctx, "Load concurrency error, will retry serially", err) + + // Run serially by calling runPiped. + stdout.Reset() + stderr.Reset() + friendlyErr, err = runner.runPiped(ctx, inv, stdout, stderr) } - event.Error(ctx, "Load concurrency error, will retry serially", err) - // Run serially by calling runPiped. - stdout.Reset() - stderr.Reset() - friendlyErr, err = runner.runPiped(ctx, inv, stdout, stderr) return stdout, stderr, friendlyErr, err } +// Postcondition: both error results have same nilness. func (runner *Runner) runConcurrent(ctx context.Context, inv Invocation) (*bytes.Buffer, *bytes.Buffer, error, error) { // Wait for 1 worker to become available. select { case <-ctx.Done(): - return nil, nil, nil, ctx.Err() + return nil, nil, ctx.Err(), ctx.Err() case runner.inFlight <- struct{}{}: defer func() { <-runner.inFlight }() } @@ -121,6 +123,7 @@ func (runner *Runner) runConcurrent(ctx context.Context, inv Invocation) (*bytes return stdout, stderr, friendlyErr, err } +// Postcondition: both error results have same nilness. func (runner *Runner) runPiped(ctx context.Context, inv Invocation, stdout, stderr io.Writer) (error, error) { // Make sure the runner is always initialized. runner.initialize() @@ -129,7 +132,7 @@ func (runner *Runner) runPiped(ctx context.Context, inv Invocation, stdout, stde // runPiped commands. select { case <-ctx.Done(): - return nil, ctx.Err() + return ctx.Err(), ctx.Err() case runner.serialized <- struct{}{}: defer func() { <-runner.serialized }() } @@ -139,7 +142,7 @@ func (runner *Runner) runPiped(ctx context.Context, inv Invocation, stdout, stde for i := 0; i < maxInFlight; i++ { select { case <-ctx.Done(): - return nil, ctx.Err() + return ctx.Err(), ctx.Err() case runner.inFlight <- struct{}{}: // Make sure we always "return" any workers we took. defer func() { <-runner.inFlight }() @@ -172,6 +175,7 @@ type Invocation struct { Logf func(format string, args ...interface{}) } +// Postcondition: both error results have same nilness. func (i *Invocation) runWithFriendlyError(ctx context.Context, stdout, stderr io.Writer) (friendlyError error, rawError error) { rawError = i.run(ctx, stdout, stderr) if rawError != nil { diff --git a/vendor/golang.org/x/tools/internal/typesinternal/objectpath.go b/vendor/golang.org/x/tools/internal/typesinternal/objectpath.go deleted file mode 100644 index 5e96e89557..0000000000 --- a/vendor/golang.org/x/tools/internal/typesinternal/objectpath.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package typesinternal - -import "go/types" - -// This file contains back doors that allow gopls to avoid method sorting when -// using the objectpath package. -// -// This is performance-critical in certain repositories, but changing the -// behavior of the objectpath package is still being discussed in -// golang/go#61443. If we decide to remove the sorting in objectpath we can -// simply delete these back doors. Otherwise, we should add a new API to -// objectpath that allows controlling the sorting. - -// SkipEncoderMethodSorting marks enc (which must be an *objectpath.Encoder) as -// not requiring sorted methods. -var SkipEncoderMethodSorting func(enc interface{}) - -// ObjectpathObject is like objectpath.Object, but allows suppressing method -// sorting. -var ObjectpathObject func(pkg *types.Package, p string, skipMethodSorting bool) (types.Object, error) diff --git a/vendor/modules.txt b/vendor/modules.txt index 47f368782c..b296b5e657 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -619,6 +619,9 @@ github.com/mattn/go-isatty # github.com/matttproud/golang_protobuf_extensions v1.0.4 ## explicit; go 1.9 github.com/matttproud/golang_protobuf_extensions/pbutil +# github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 +## explicit; go 1.19 +github.com/matttproud/golang_protobuf_extensions/v2/pbutil # github.com/miekg/dns v1.1.56 ## explicit; go 1.19 github.com/miekg/dns @@ -753,8 +756,8 @@ github.com/prometheus/client_golang/prometheus/testutil/promlint # github.com/prometheus/client_model v0.5.0 ## explicit; go 1.19 github.com/prometheus/client_model/go -# github.com/prometheus/common v0.44.0 -## explicit; go 1.18 +# github.com/prometheus/common v0.45.0 +## explicit; go 1.20 github.com/prometheus/common/config github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg @@ -772,7 +775,7 @@ github.com/prometheus/exporter-toolkit/web github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v0.48.0 +# github.com/prometheus/prometheus v0.48.1-0.20231201222638-e4ec263bcc11 ## explicit; go 1.20 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -906,7 +909,7 @@ github.com/thanos-io/promql-engine/execution/warnings github.com/thanos-io/promql-engine/extlabels github.com/thanos-io/promql-engine/logicalplan github.com/thanos-io/promql-engine/query -# github.com/thanos-io/thanos v0.32.5-0.20231127170340-8ffb9da1383e +# github.com/thanos-io/thanos v0.32.5-0.20231204192512-28407d61e72d ## explicit; go 1.21 github.com/thanos-io/thanos/pkg/block github.com/thanos-io/thanos/pkg/block/indexheader @@ -1060,7 +1063,10 @@ go.opencensus.io/trace go.opencensus.io/trace/internal go.opencensus.io/trace/propagation go.opencensus.io/trace/tracestate -# go.opentelemetry.io/collector/pdata v1.0.0-rcv0016 +# go.opentelemetry.io/collector/featuregate v0.77.0 +## explicit; go 1.19 +go.opentelemetry.io/collector/featuregate +# go.opentelemetry.io/collector/pdata v1.0.0-rcv0017 ## explicit; go 1.20 go.opentelemetry.io/collector/pdata/internal go.opentelemetry.io/collector/pdata/internal/data @@ -1077,7 +1083,7 @@ go.opentelemetry.io/collector/pdata/internal/otlp go.opentelemetry.io/collector/pdata/pcommon go.opentelemetry.io/collector/pdata/pmetric go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp -# go.opentelemetry.io/collector/semconv v0.87.0 +# go.opentelemetry.io/collector/semconv v0.88.0 ## explicit; go 1.20 go.opentelemetry.io/collector/semconv/v1.6.1 # go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 @@ -1189,11 +1195,11 @@ golang.org/x/crypto/internal/alias golang.org/x/crypto/internal/poly1305 golang.org/x/crypto/pkcs12 golang.org/x/crypto/pkcs12/internal/rc2 -# golang.org/x/exp v0.0.0-20231006140011-7918f672742d +# golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa ## explicit; go 1.20 golang.org/x/exp/constraints golang.org/x/exp/slices -# golang.org/x/mod v0.13.0 +# golang.org/x/mod v0.14.0 ## explicit; go 1.18 golang.org/x/mod/semver # golang.org/x/net v0.18.0 @@ -1252,7 +1258,7 @@ golang.org/x/text/unicode/norm # golang.org/x/time v0.4.0 ## explicit; go 1.18 golang.org/x/time/rate -# golang.org/x/tools v0.14.0 +# golang.org/x/tools v0.15.0 ## explicit; go 1.18 golang.org/x/tools/go/gcexportdata golang.org/x/tools/go/internal/packagesdriver