diff --git a/operations/pyroscope/helm/pyroscope/rendered/micro-services-hpa.yaml b/operations/pyroscope/helm/pyroscope/rendered/micro-services-hpa.yaml index 8a706e88ef..396e644416 100644 --- a/operations/pyroscope/helm/pyroscope/rendered/micro-services-hpa.yaml +++ b/operations/pyroscope/helm/pyroscope/rendered/micro-services-hpa.yaml @@ -2134,6 +2134,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2227,6 +2234,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2320,6 +2334,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2413,6 +2434,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2807,6 +2835,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2908,6 +2943,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -3005,6 +3047,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 diff --git a/operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml b/operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml index 1a58cd3160..d9442c5cf5 100644 --- a/operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml +++ b/operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml @@ -2279,6 +2279,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2373,6 +2380,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2467,6 +2481,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2561,6 +2582,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2655,6 +2683,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -2749,6 +2784,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -3028,6 +3070,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -3129,6 +3178,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 @@ -3226,6 +3282,13 @@ spec: - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" - "-store-gateway.sharding-ring.replication-factor=3" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 diff --git a/operations/pyroscope/helm/pyroscope/rendered/single-binary.yaml b/operations/pyroscope/helm/pyroscope/rendered/single-binary.yaml index c85ac1b5d4..86ca35e594 100644 --- a/operations/pyroscope/helm/pyroscope/rendered/single-binary.yaml +++ b/operations/pyroscope/helm/pyroscope/rendered/single-binary.yaml @@ -1376,6 +1376,13 @@ spec: - "-config.file=/etc/pyroscope/config.yaml" - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml" - "-log.level=debug" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "default.svc.cluster.local." ports: - name: http2 containerPort: 4040 diff --git a/operations/pyroscope/helm/pyroscope/templates/deployments-statefulsets.yaml b/operations/pyroscope/helm/pyroscope/templates/deployments-statefulsets.yaml index b6162ab30a..026e9f2b53 100644 --- a/operations/pyroscope/helm/pyroscope/templates/deployments-statefulsets.yaml +++ b/operations/pyroscope/helm/pyroscope/templates/deployments-statefulsets.yaml @@ -70,8 +70,14 @@ spec: {{- range $key, $value := $extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} - {{- with $values.extraEnvVars }} env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_FQDN + value: "{{ .Release.Namespace }}.svc{{ .Values.pyroscope.cluster_domain }}" + {{- with $values.extraEnvVars }} {{- range $key, $value := . }} - name: {{ $key }} {{- if kindIs "map" $value }} diff --git a/operations/pyroscope/helm/pyroscope/templates/services.yaml b/operations/pyroscope/helm/pyroscope/templates/services.yaml index d20d2a5aae..193c7419af 100644 --- a/operations/pyroscope/helm/pyroscope/templates/services.yaml +++ b/operations/pyroscope/helm/pyroscope/templates/services.yaml @@ -22,6 +22,9 @@ spec: targetPort: {{ $values.service.port_name }} protocol: TCP name: {{ $values.service.port_name }} + {{- with $values.service.extraPorts }} + {{- toYaml . | nindent 4 }} + {{- end }} selector: {{- include "pyroscope.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: {{ $component | quote }} @@ -45,6 +48,12 @@ spec: targetPort: {{ $values.service.port_name }} protocol: TCP name: {{ $values.service.port_name }} + {{- with $values.service.extraPorts }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if $values.service.publishNotReadyAddresses }} + publishNotReadyAddresses: true + {{- end }} selector: {{- include "pyroscope.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: {{ $component | quote }} diff --git a/pkg/api/api.go b/pkg/api/api.go index ea97f6e69a..230072ca14 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -22,6 +22,11 @@ import ( "github.com/grafana/dskit/server" grpcgw "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + compactorv1 "github.com/grafana/pyroscope/api/gen/proto/go/compactor/v1" + metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" + querybackendv1 "github.com/grafana/pyroscope/api/gen/proto/go/querybackend/v1" + "github.com/grafana/pyroscope/pkg/experiment/metastore" + "github.com/grafana/pyroscope/pkg/experiment/querybackend" "github.com/grafana/pyroscope/public" "github.com/grafana/pyroscope/api/gen/proto/go/adhocprofiles/v1/adhocprofilesv1connect" @@ -320,6 +325,15 @@ func (a *API) RegisterAdHocProfiles(ahp *adhocprofiles.AdHocProfiles) { adhocprofilesv1connect.RegisterAdHocProfileServiceHandler(a.server.HTTP, ahp, a.connectOptionsAuthRecovery()...) } +func (a *API) RegisterMetastore(svc *metastore.Metastore) { + metastorev1.RegisterMetastoreServiceServer(a.server.GRPC, svc) + compactorv1.RegisterCompactionPlannerServer(a.server.GRPC, svc) +} + +func (a *API) RegisterQueryBackend(svc *querybackend.QueryBackend) { + querybackendv1.RegisterQueryBackendServiceServer(a.server.GRPC, svc) +} + func (a *API) connectOptionsRecovery() []connect.HandlerOption { return append(connectapi.DefaultHandlerOptions(), a.recoveryMiddleware) } diff --git a/pkg/experiment/compactor/compaction_worker.go b/pkg/experiment/compactor/compaction_worker.go index 5a8964aaf2..c961d32ced 100644 --- a/pkg/experiment/compactor/compaction_worker.go +++ b/pkg/experiment/compactor/compaction_worker.go @@ -50,9 +50,9 @@ type Config struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { const prefix = "compaction-worker." tempdir := filepath.Join(os.TempDir(), "pyroscope-compactor") - f.IntVar(&cfg.JobCapacity, prefix+"job-capacity", 3, "how many concurrent jobs will a worker run at most") - f.IntVar(&cfg.SmallObjectSize, prefix+"small-object-size-bytes", 8<<20, "size of the object that can be loaded in memory") - f.StringVar(&cfg.TempDir, prefix+"temp-dir", tempdir, "temporary directory for compaction jobs") + f.IntVar(&cfg.JobCapacity, prefix+"job-capacity", 3, "How many concurrent jobs will a worker run at most.") + f.IntVar(&cfg.SmallObjectSize, prefix+"small-object-size-bytes", 8<<20, "Size of the object that can be loaded in memory.") + f.StringVar(&cfg.TempDir, prefix+"temp-dir", tempdir, "Temporary directory for compaction jobs.") } func New(config Config, logger log.Logger, metastoreClient *metastoreclient.Client, storage objstore.Bucket, reg prometheus.Registerer) (*Worker, error) { diff --git a/pkg/experiment/ingester/segment.go b/pkg/experiment/ingester/segment.go index 2043a8b664..bcd1ff9890 100644 --- a/pkg/experiment/ingester/segment.go +++ b/pkg/experiment/ingester/segment.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "encoding/json" + "flag" "fmt" "os" "path" @@ -39,6 +40,14 @@ const pathSegments = "segments" const pathAnon = tenant.DefaultTenantID const pathBlock = "block.bin" +type Config struct{} + +func (cfg *Config) RegisterFlags(f *flag.FlagSet) {} + +type SegmentWriter struct { + // TODO: Implement +} + type shardKey uint32 type segmentsWriter struct { diff --git a/pkg/experiment/metastore/client/client.go b/pkg/experiment/metastore/client/client.go index 7a6a14d16e..8dd198b6ac 100644 --- a/pkg/experiment/metastore/client/client.go +++ b/pkg/experiment/metastore/client/client.go @@ -2,9 +2,6 @@ package metastoreclient import ( "context" - "flag" - "fmt" - "os" "github.com/go-kit/log" @@ -18,52 +15,35 @@ import ( metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" ) -type Config struct { - MetastoreAddress string `yaml:"address"` - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate between the query-frontends and the query-schedulers."` -} - -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.MetastoreAddress, "metastore.address", "localhost:9095", "") - cfg.GRPCClientConfig.RegisterFlagsWithPrefix("metastore.grpc-client-config", f) -} - -func (cfg *Config) Validate() error { - if cfg.MetastoreAddress == "" { - return fmt.Errorf("metastore.address is required") - } - return cfg.GRPCClientConfig.Validate() -} - type Client struct { metastorev1.MetastoreServiceClient compactorv1.CompactionPlannerClient service services.Service conn *grpc.ClientConn - config Config } -func New(config Config, logger log.Logger) (c *Client, err error) { - c = &Client{config: config} - c.conn, err = dial(c.config, logger) +func New(address string, grpcClientConfig grpcclient.Config, logger log.Logger) (*Client, error) { + conn, err := dial(address, grpcClientConfig, logger) if err != nil { return nil, err } - c.MetastoreServiceClient = metastorev1.NewMetastoreServiceClient(c.conn) - c.CompactionPlannerClient = compactorv1.NewCompactionPlannerClient(c.conn) + var c Client + c.MetastoreServiceClient = metastorev1.NewMetastoreServiceClient(conn) + c.CompactionPlannerClient = compactorv1.NewCompactionPlannerClient(conn) c.service = services.NewIdleService(c.starting, c.stopping) - return c, nil + c.conn = conn + return &c, nil } func (c *Client) Service() services.Service { return c.service } func (c *Client) starting(context.Context) error { return nil } func (c *Client) stopping(error) error { return c.conn.Close() } -func dial(cfg Config, logger log.Logger) (*grpc.ClientConn, error) { - if err := cfg.Validate(); err != nil { +func dial(address string, grpcClientConfig grpcclient.Config, _ log.Logger) (*grpc.ClientConn, error) { + if err := grpcClientConfig.Validate(); err != nil { return nil, err } - options, err := cfg.GRPCClientConfig.DialOption(nil, nil) + options, err := grpcClientConfig.DialOption(nil, nil) if err != nil { return nil, err } @@ -72,16 +52,18 @@ func dial(cfg Config, logger log.Logger) (*grpc.ClientConn, error) { grpc.WithDefaultServiceConfig(grpcServiceConfig), grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer())), ) - if os.Getenv("KUBERNETES_SERVICE_HOST") != "" { - builder, err := NewGrpcResolverBuilder(logger, cfg.MetastoreAddress) - if err != nil { - return nil, fmt.Errorf("failed to create grpc resolver builder: %w", err) - } - options = append(options, grpc.WithResolvers(builder)) - return grpc.Dial(builder.resolverAddrStub(), options...) - } else { - return grpc.Dial(cfg.MetastoreAddress, options...) - } + // TODO: Implement k8s grpc resolver. + // Note that this may require additional permissions. + // Consider: https://github.com/sercand/kuberesolver + // if os.Getenv("KUBERNETES_SERVICE_HOST") != "" { + // builder, err := NewGrpcResolverBuilder(logger, address) + // if err != nil { + // return nil, fmt.Errorf("failed to create grpc resolver builder: %w", err) + // } + // options = append(options, grpc.WithResolvers(builder)) + // return grpc.Dial(builder.resolverAddrStub(), options...) + // } + return grpc.Dial(address, options...) } const grpcServiceConfig = `{ diff --git a/pkg/experiment/metastore/metastore.go b/pkg/experiment/metastore/metastore.go index 7ccc51aa40..063983a3c2 100644 --- a/pkg/experiment/metastore/metastore.go +++ b/pkg/experiment/metastore/metastore.go @@ -14,6 +14,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/grafana/dskit/flagext" + "github.com/grafana/dskit/grpcclient" "github.com/grafana/dskit/services" "github.com/hashicorp/raft" raftwal "github.com/hashicorp/raft-wal" @@ -41,8 +42,10 @@ const ( ) type Config struct { - DataDir string `yaml:"data_dir"` - Raft RaftConfig `yaml:"raft"` + Address string `yaml:"address"` + GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate with the metastore."` + DataDir string `yaml:"data_dir"` + Raft RaftConfig `yaml:"raft"` } type RaftConfig struct { @@ -60,12 +63,13 @@ type RaftConfig struct { func (cfg *Config) RegisterFlags(f *flag.FlagSet) { const prefix = "metastore." + f.StringVar(&cfg.Address, prefix+"address", "localhost:9095", "") + cfg.GRPCClientConfig.RegisterFlagsWithPrefix(prefix+"grpc-client-config", f) f.StringVar(&cfg.DataDir, prefix+"data-dir", "./data-metastore/data", "") - cfg.Raft.RegisterFlags(f) + cfg.Raft.RegisterFlagsWithPrefix(prefix+"raft.", f) } -func (cfg *RaftConfig) RegisterFlags(f *flag.FlagSet) { - const prefix = "metastore.raft." +func (cfg *RaftConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.Dir, prefix+"dir", "./data-metastore/raft", "") f.Var((*flagext.StringSlice)(&cfg.BootstrapPeers), prefix+"bootstrap-peers", "") f.IntVar(&cfg.BootstrapExpectPeers, prefix+"bootstrap-expect-peers", 1, "Expected number of peers including the local node.") diff --git a/pkg/phlare/modules.go b/pkg/phlare/modules.go index b8cd26563b..30fdb3a9d3 100644 --- a/pkg/phlare/modules.go +++ b/pkg/phlare/modules.go @@ -80,9 +80,15 @@ const ( TenantSettings string = "tenant-settings" AdHocProfiles string = "ad-hoc-profiles" - // QueryFrontendTripperware string = "query-frontend-tripperware" - // IndexGateway string = "index-gateway" - // IndexGatewayRing string = "index-gateway-ring" + // Experimental modules + + Metastore string = "metastore" + MetastoreClient string = "metastore-client" + SegmentWriter string = "segment-writer" + QueryBackend string = "query-backend" + QueryBackendClient string = "query-backend-client" + CompactionWorker string = "compaction-worker" + HealthService string = "health-service" ) var objectStoreTypeStats = usagestats.NewString("store_object_type") diff --git a/pkg/phlare/modules_experimental.go b/pkg/phlare/modules_experimental.go new file mode 100644 index 0000000000..2df3c2db55 --- /dev/null +++ b/pkg/phlare/modules_experimental.go @@ -0,0 +1,83 @@ +package phlare + +import ( + "context" + + "github.com/go-kit/log" + "github.com/grafana/dskit/services" + "google.golang.org/grpc/health/grpc_health_v1" + + compactionworker "github.com/grafana/pyroscope/pkg/experiment/compactor" + "github.com/grafana/pyroscope/pkg/experiment/metastore" + metastoreclient "github.com/grafana/pyroscope/pkg/experiment/metastore/client" + "github.com/grafana/pyroscope/pkg/experiment/querybackend" + querybackendclient "github.com/grafana/pyroscope/pkg/experiment/querybackend/client" + "github.com/grafana/pyroscope/pkg/util/health" +) + +func (f *Phlare) initSegmentWriter() (services.Service, error) { + // TODO(kolesnikovae): initialize the component. + return services.NewIdleService( + func(context.Context) error { return nil }, + func(error) error { return nil }, + ), nil +} + +func (f *Phlare) initCompactionWorker() (svc services.Service, err error) { + logger := log.With(f.logger, "component", "compaction-worker") + f.compactionWorker, err = compactionworker.New(f.Cfg.CompactionWorker, logger, f.metastoreClient, f.storageBucket, f.reg) + if err != nil { + return nil, err + } + return f.compactionWorker, nil +} + +func (f *Phlare) initMetastore() (services.Service, error) { + logger := log.With(f.logger, "component", "metastore") + m, err := metastore.New(f.Cfg.Metastore, f.TenantLimits, logger, f.reg, f.healthService, f.metastoreClient) + if err != nil { + return nil, err + } + f.API.RegisterMetastore(m) + f.metastore = m + return m.Service(), nil +} + +func (f *Phlare) initMetastoreClient() (services.Service, error) { + mc, err := metastoreclient.New(f.Cfg.Metastore.Address, f.Cfg.Metastore.GRPCClientConfig, f.logger) + if err != nil { + return nil, err + } + f.metastoreClient = mc + return mc.Service(), nil +} + +func (f *Phlare) initQueryBackend() (services.Service, error) { + br := querybackend.NewBlockReader(f.logger, f.storageBucket) + logger := log.With(f.logger, "component", "query-backend") + b, err := querybackend.New(f.Cfg.QueryBackend, logger, f.reg, f.queryBackendClient, br) + if err != nil { + return nil, err + } + f.API.RegisterQueryBackend(b) + return b.Service(), nil +} + +func (f *Phlare) initQueryBackendClient() (services.Service, error) { + c, err := querybackendclient.New( + f.Cfg.QueryBackend.Address, + f.Cfg.QueryBackend.GRPCClientConfig, + ) + if err != nil { + return nil, err + } + f.queryBackendClient = c + return c.Service(), nil +} + +func (f *Phlare) initHealthService() (services.Service, error) { + healthService := health.NewGRPCHealthService() + grpc_health_v1.RegisterHealthServer(f.Server.GRPC, healthService) + f.healthService = healthService + return healthService, nil +} diff --git a/pkg/phlare/phlare.go b/pkg/phlare/phlare.go index df60bbd64b..3e7df24897 100644 --- a/pkg/phlare/phlare.go +++ b/pkg/phlare/phlare.go @@ -42,6 +42,12 @@ import ( "github.com/grafana/pyroscope/pkg/cfg" "github.com/grafana/pyroscope/pkg/compactor" "github.com/grafana/pyroscope/pkg/distributor" + compactionworker "github.com/grafana/pyroscope/pkg/experiment/compactor" + segmentwriter "github.com/grafana/pyroscope/pkg/experiment/ingester" + "github.com/grafana/pyroscope/pkg/experiment/metastore" + metastoreclient "github.com/grafana/pyroscope/pkg/experiment/metastore/client" + "github.com/grafana/pyroscope/pkg/experiment/querybackend" + querybackendclient "github.com/grafana/pyroscope/pkg/experiment/querybackend/client" "github.com/grafana/pyroscope/pkg/frontend" "github.com/grafana/pyroscope/pkg/ingester" phlareobj "github.com/grafana/pyroscope/pkg/objstore" @@ -59,6 +65,7 @@ import ( "github.com/grafana/pyroscope/pkg/usagestats" "github.com/grafana/pyroscope/pkg/util" "github.com/grafana/pyroscope/pkg/util/cli" + "github.com/grafana/pyroscope/pkg/util/health" "github.com/grafana/pyroscope/pkg/validation" "github.com/grafana/pyroscope/pkg/validation/exporter" ) @@ -91,6 +98,16 @@ type Config struct { ConfigFile string `yaml:"-"` ConfigExpandEnv bool `yaml:"-"` + + // Experimental modules. + // TODO(kolesnikovae): + // - Generalized experimental features? + // - Better naming. + v2Experiment bool + SegmentWriter segmentwriter.Config `yaml:"segment_writer" doc:"hidden"` + Metastore metastore.Config `yaml:"metastore" doc:"hidden"` + QueryBackend querybackend.Config `yaml:"query_backend" doc:"hidden"` + CompactionWorker compactionworker.Config `yaml:"compaction_worker" doc:"hidden"` } func newDefaultConfig() *Config { @@ -149,6 +166,14 @@ func (c *Config) RegisterFlagsWithContext(ctx context.Context, f *flag.FlagSet) c.LimitsConfig.RegisterFlags(f) c.Compactor.RegisterFlags(f, log.NewLogfmtLogger(os.Stderr)) c.API.RegisterFlags(f) + + c.v2Experiment = os.Getenv("PYROSCOPE_V2_EXPERIMENT") != "" + if c.v2Experiment { + c.Metastore.RegisterFlags(f) + c.SegmentWriter.RegisterFlags(f) + c.QueryBackend.RegisterFlags(f) + c.CompactionWorker.RegisterFlags(f) + } } // registerServerFlagsWithChangedDefaultValues registers *Config.Server flags, but overrides some defaults set by the weaveworks package. @@ -242,6 +267,17 @@ type Phlare struct { auth connect.Option ingester *ingester.Ingester frontend *frontend.Frontend + + // Experimental modules. + //nolint:unused + segmentWriter *segmentwriter.SegmentWriter + metastore *metastore.Metastore + metastoreClient *metastoreclient.Client + //nolint:unused + queryBackend *querybackend.QueryBackend + queryBackendClient *querybackendclient.Client + compactionWorker *compactionworker.Worker + healthService health.Service } func New(cfg Config) (*Phlare, error) { @@ -250,9 +286,10 @@ func New(cfg Config) (*Phlare, error) { usagestats.Edition("oss") phlare := &Phlare{ - Cfg: cfg, - logger: logger, - reg: prometheus.DefaultRegisterer, + Cfg: cfg, + logger: logger, + reg: prometheus.DefaultRegisterer, + healthService: health.NoOpService, } if err := cfg.Validate(); err != nil { return nil, err @@ -311,7 +348,7 @@ func (f *Phlare) setupModuleManager() error { // Add dependencies deps := map[string][]string{ - All: {Ingester, Distributor, QueryScheduler, QueryFrontend, Querier, StoreGateway, Admin, TenantSettings, Compactor, AdHocProfiles}, + All: {Ingester, Distributor, QueryFrontend, QueryScheduler, Querier, StoreGateway, Compactor, Admin, TenantSettings, AdHocProfiles}, Server: {GRPCGateway}, API: {Server}, @@ -334,6 +371,32 @@ func (f *Phlare) setupModuleManager() error { AdHocProfiles: {API, Overrides, Storage}, } + // Experimental modules. + if f.Cfg.v2Experiment { + experimentalModules := map[string][]string{ + SegmentWriter: {Overrides, API, MemberlistKV, Storage, UsageReport, MetastoreClient}, + Metastore: {Overrides, API, HealthService, MetastoreClient}, + CompactionWorker: {Overrides, API, Storage, Overrides, MetastoreClient}, + QueryBackend: {Overrides, API, Storage, Overrides, QueryBackendClient}, + HealthService: {Overrides, API}, + } + for k, v := range experimentalModules { + deps[k] = v + } + + // TODO(kolesnikovae): Inject new distributor dependencies, if any. + deps[All] = append(deps[All], SegmentWriter, Metastore, CompactionWorker, QueryBackend, HealthService) + deps[QueryFrontend] = append(deps[QueryFrontend], MetastoreClient, QueryBackendClient) + + mm.RegisterModule(SegmentWriter, f.initSegmentWriter) + mm.RegisterModule(Metastore, f.initMetastore) + mm.RegisterModule(CompactionWorker, f.initCompactionWorker) + mm.RegisterModule(QueryBackend, f.initQueryBackend) + mm.RegisterModule(MetastoreClient, f.initMetastoreClient, modules.UserInvisibleModule) + mm.RegisterModule(QueryBackendClient, f.initQueryBackendClient, modules.UserInvisibleModule) + mm.RegisterModule(HealthService, f.initHealthService, modules.UserInvisibleModule) + } + for mod, targets := range deps { if err := mm.AddDependency(mod, targets...); err != nil { return err diff --git a/tools/dev/experiment/local_install.sh b/tools/dev/experiment/local_install.sh new file mode 100755 index 0000000000..8a6bfe8f51 --- /dev/null +++ b/tools/dev/experiment/local_install.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -x +set -e + +if [ -z "$DOCKER_REPO" ]; then + echo "Specify the docker repository using the DOCKER_REPO variable." + exit 1 +fi + +IMAGE_NAME=$DOCKER_REPO/pyroscope +PYROSCOPE_TEST_NAMESPACE=pyroscope-test +HELM_CHART=./operations/pyroscope/helm/pyroscope +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALUES_FILE="$SCRIPT_DIR/values-micro-services-experiment.yaml" + +# build and push image + +GOOS=linux GOARCH=amd64 IMAGE_PREFIX="$DOCKER_REPO/" make docker-image/pyroscope/push + +IMAGE_TAG=$(docker image ls | grep "$IMAGE_NAME" | head -n 1 | awk '{print $2}') +if [ -z "$IMAGE_TAG" ]; then + echo "Error: can't find image tag for $IMAGE_NAME." + exit 1 +fi + +echo "using image: $IMAGE_NAME:$IMAGE_TAG" + +# deploy + +helm -n "$PYROSCOPE_TEST_NAMESPACE" upgrade --install \ + --create-namespace pyroscope \ + --values "$VALUES_FILE" \ + --set pyroscope.image.repository="$IMAGE_NAME" \ + --set pyroscope.image.tag="$IMAGE_TAG" \ + "$HELM_CHART" + +kubectl --namespace "$PYROSCOPE_TEST_NAMESPACE" port-forward svc/pyroscope-query-frontend 4040:4040 diff --git a/tools/dev/experiment/local_uninstall.sh b/tools/dev/experiment/local_uninstall.sh new file mode 100755 index 0000000000..60d88b776a --- /dev/null +++ b/tools/dev/experiment/local_uninstall.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +PYROSCOPE_TEST_NAMESPACE=pyroscope-test + +helm -n "$PYROSCOPE_TEST_NAMESPACE" uninstall pyroscope diff --git a/tools/dev/experiment/values-micro-services-experiment.yaml b/tools/dev/experiment/values-micro-services-experiment.yaml new file mode 100644 index 0000000000..f7782d25fc --- /dev/null +++ b/tools/dev/experiment/values-micro-services-experiment.yaml @@ -0,0 +1,71 @@ +pyroscope: + extraEnvVars: + PYROSCOPE_V2_EXPERIMENT: 1 + extraArgs: + query-backend.address: "dns:///_grpc._tcp.pyroscope-query-worker-headless.$(NAMESPACE_FQDN):9095" + metastore.address: "dns:///_grpc._tcp.pyroscope-metastore-headless.$(NAMESPACE_FQDN):9095" + metastore.raft.bind-address: ":9099" + metastore.raft.server-id: "$(POD_NAME).pyroscope-metastore-headless.$(NAMESPACE_FQDN)" + metastore.raft.advertise-address: "$(POD_NAME).pyroscope-metastore-headless.$(NAMESPACE_FQDN):9099" + metastore.raft.bootstrap-peers: "dnssrvnoa+_raft._tcp.pyroscope-metastore-headless.$(NAMESPACE_FQDN):9099" + metastore.raft.bootstrap-expect-peers: "3" + + components: + distributor: + kind: Deployment + replicaCount: 2 + + segment-writer: + kind: StatefulSet + replicaCount: 3 + persistence: + enabled: false + + query-frontend: + kind: Deployment + replicaCount: 2 + + query-backend: + kind: Deployment + replicaCount: 3 + service: + extraPorts: + - name: grpc + port: 9095 + protocol: TCP + targetPort: 9095 + + metastore: + kind: StatefulSet + replicaCount: 3 + service: + publishNotReadyAddresses: true + extraPorts: + - name: grpc + port: 9095 + protocol: TCP + targetPort: 9095 + - name: raft + port: 9099 + protocol: TCP + targetPort: 9099 + persistence: + enabled: true + size: 10Gi + extraVolumeMounts: + - name: data + mountPath: /data-metastore + subPath: metastore + + compaction-worker: + kind: StatefulSet + replicaCount: 3 + persistence: + enabled: false + extraVolumeMounts: + - name: data + mountPath: /data-compaction-worker + subPath: compaction-worker + +minio: + enabled: true