chore: add experimental modules (#3480)

grafana · Aug 14, 2024 · 04923f5 · 04923f5
1 parent b16b27a
commit 04923f5
Show file tree

Hide file tree

Showing 16 changed files with 468 additions and 56 deletions.
diff --git a/operations/pyroscope/helm/pyroscope/rendered/micro-services-hpa.yaml b/operations/pyroscope/helm/pyroscope/rendered/micro-services-hpa.yaml
@@ -2134,6 +2134,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2227,6 +2234,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2320,6 +2334,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2413,6 +2434,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2807,6 +2835,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2908,6 +2943,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -3005,6 +3047,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040

diff --git a/operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml b/operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml
@@ -2279,6 +2279,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2373,6 +2380,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2467,6 +2481,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2561,6 +2582,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2655,6 +2683,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -2749,6 +2784,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -3028,6 +3070,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -3129,6 +3178,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040
@@ -3226,6 +3282,13 @@ spec:
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
             - "-store-gateway.sharding-ring.replication-factor=3"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040

diff --git a/operations/pyroscope/helm/pyroscope/rendered/single-binary.yaml b/operations/pyroscope/helm/pyroscope/rendered/single-binary.yaml
@@ -1376,6 +1376,13 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "default.svc.cluster.local."
           ports:
             - name: http2
               containerPort: 4040

diff --git a/operations/pyroscope/helm/pyroscope/templates/deployments-statefulsets.yaml b/operations/pyroscope/helm/pyroscope/templates/deployments-statefulsets.yaml
@@ -70,8 +70,14 @@ spec:
           {{- range $key, $value := $extraArgs }}
             - "-{{ $key }}={{ $value }}"
           {{- end }}
-          {{- with $values.extraEnvVars }}
           env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_FQDN
+              value: "{{ .Release.Namespace }}.svc{{ .Values.pyroscope.cluster_domain }}"
+          {{- with $values.extraEnvVars }}
           {{- range $key, $value := . }}
             - name: {{ $key }}
               {{- if kindIs "map" $value }}

diff --git a/operations/pyroscope/helm/pyroscope/templates/services.yaml b/operations/pyroscope/helm/pyroscope/templates/services.yaml
@@ -22,6 +22,9 @@ spec:
       targetPort: {{ $values.service.port_name }}
       protocol: TCP
       name: {{ $values.service.port_name }}
+    {{- with $values.service.extraPorts }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   selector:
     {{- include "pyroscope.selectorLabels" . | nindent 4 }}
     app.kubernetes.io/component: {{ $component | quote }}
@@ -45,6 +48,12 @@ spec:
       targetPort: {{ $values.service.port_name }}
       protocol: TCP
       name: {{ $values.service.port_name }}
+    {{- with $values.service.extraPorts }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- if $values.service.publishNotReadyAddresses }}
+  publishNotReadyAddresses: true
+  {{- end }}
   selector:
     {{- include "pyroscope.selectorLabels" . | nindent 4 }}
     app.kubernetes.io/component: {{ $component | quote }}

diff --git a/pkg/api/api.go b/pkg/api/api.go
@@ -22,6 +22,11 @@ import (
 	"github.com/grafana/dskit/server"
 	grpcgw "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 
+	compactorv1 "github.com/grafana/pyroscope/api/gen/proto/go/compactor/v1"
+	metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1"
+	querybackendv1 "github.com/grafana/pyroscope/api/gen/proto/go/querybackend/v1"
+	"github.com/grafana/pyroscope/pkg/experiment/metastore"
+	"github.com/grafana/pyroscope/pkg/experiment/querybackend"
 	"github.com/grafana/pyroscope/public"
 
 	"github.com/grafana/pyroscope/api/gen/proto/go/adhocprofiles/v1/adhocprofilesv1connect"
@@ -320,6 +325,15 @@ func (a *API) RegisterAdHocProfiles(ahp *adhocprofiles.AdHocProfiles) {
 	adhocprofilesv1connect.RegisterAdHocProfileServiceHandler(a.server.HTTP, ahp, a.connectOptionsAuthRecovery()...)
 }
 
+func (a *API) RegisterMetastore(svc *metastore.Metastore) {
+	metastorev1.RegisterMetastoreServiceServer(a.server.GRPC, svc)
+	compactorv1.RegisterCompactionPlannerServer(a.server.GRPC, svc)
+}
+
+func (a *API) RegisterQueryBackend(svc *querybackend.QueryBackend) {
+	querybackendv1.RegisterQueryBackendServiceServer(a.server.GRPC, svc)
+}
+
 func (a *API) connectOptionsRecovery() []connect.HandlerOption {
 	return append(connectapi.DefaultHandlerOptions(), a.recoveryMiddleware)
 }

diff --git a/pkg/experiment/compactor/compaction_worker.go b/pkg/experiment/compactor/compaction_worker.go
@@ -50,9 +50,9 @@ type Config struct {
 func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	const prefix = "compaction-worker."
 	tempdir := filepath.Join(os.TempDir(), "pyroscope-compactor")
-	f.IntVar(&cfg.JobCapacity, prefix+"job-capacity", 3, "how many concurrent jobs will a worker run at most")
-	f.IntVar(&cfg.SmallObjectSize, prefix+"small-object-size-bytes", 8<<20, "size of the object that can be loaded in memory")
-	f.StringVar(&cfg.TempDir, prefix+"temp-dir", tempdir, "temporary directory for compaction jobs")
+	f.IntVar(&cfg.JobCapacity, prefix+"job-capacity", 3, "How many concurrent jobs will a worker run at most.")
+	f.IntVar(&cfg.SmallObjectSize, prefix+"small-object-size-bytes", 8<<20, "Size of the object that can be loaded in memory.")
+	f.StringVar(&cfg.TempDir, prefix+"temp-dir", tempdir, "Temporary directory for compaction jobs.")
 }
 
 func New(config Config, logger log.Logger, metastoreClient *metastoreclient.Client, storage objstore.Bucket, reg prometheus.Registerer) (*Worker, error) {

diff --git a/pkg/experiment/ingester/segment.go b/pkg/experiment/ingester/segment.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"os"
 	"path"
@@ -39,6 +40,14 @@ const pathSegments = "segments"
 const pathAnon = tenant.DefaultTenantID
 const pathBlock = "block.bin"
 
+type Config struct{}
+
+func (cfg *Config) RegisterFlags(f *flag.FlagSet) {}
+
+type SegmentWriter struct {
+	// TODO: Implement
+}
+
 type shardKey uint32
 
 type segmentsWriter struct {