WIP: Compaction planning in metastore (#3427)

* First iteration of compaction planning in metastore * Second iteration of compaction planning in metastore * Add GetCompactionJobs back * Create and persist jobs in the same transaction as blocks * Add simple logging for compaction planning * Fix bug * Remove unused proto message * Remove unused raft command type * Add a simple config for compaction planning
grafana · Jul 16, 2024 · 8371c57 · 8371c57
1 parent 5d9a4ec
commit 8371c57
Show file tree

Hide file tree

Showing 14 changed files with 547 additions and 248 deletions.
diff --git a/api/copmactor/v1/compactor.proto → api/compactor/v1/compactor.proto b/api/copmactor/v1/compactor.proto → api/compactor/v1/compactor.proto
@@ -39,6 +39,7 @@ message CompactionJob {
   CompactionOptions options = 2;
   // List of the input blocks.
   repeated metastore.v1.BlockMeta blocks = 3;
+  CompactionJobStatus status = 4;
 }
 
 message CompactionOptions {
@@ -58,7 +59,6 @@ message UpdateJobStatusRequest {
 }
 
 message CompactionJobStatus {
-  string name = 1;
   // Status update allows the planner to keep
   // track of the job ownership and compaction
   // progress:
@@ -72,8 +72,8 @@ message CompactionJobStatus {
   //   or cancels the compaction job.
   //
   // Partial results/status is not allowed.
-  CompactionStatus status = 2;
-  CompletedJob completed_job = 3;
+  CompactionStatus status = 1;
+  CompletedJob completed_job = 2;
 }
 
 enum CompactionStatus {

diff --git a/...gen/proto/go/copmactor/v1/compactor.pb.go → ...gen/proto/go/compactor/v1/compactor.pb.go b/...gen/proto/go/copmactor/v1/compactor.pb.go → ...gen/proto/go/compactor/v1/compactor.pb.go
diff --git a/...o/go/copmactor/v1/compactor_vtproto.pb.go → ...o/go/compactor/v1/compactor_vtproto.pb.go b/...o/go/copmactor/v1/compactor_vtproto.pb.go → ...o/go/compactor/v1/compactor_vtproto.pb.go
diff --git a/...1/compactorv1connect/compactor.connect.go → ...1/compactorv1connect/compactor.connect.go b/...1/compactorv1connect/compactor.connect.go → ...1/compactorv1connect/compactor.connect.go
diff --git a/...mpactorv1connect/compactor.connect.mux.go → ...mpactorv1connect/compactor.connect.mux.go b/...mpactorv1connect/compactor.connect.mux.go → ...mpactorv1connect/compactor.connect.mux.go
diff --git a/api/openapiv2/gen/phlare.swagger.json b/api/openapiv2/gen/phlare.swagger.json
@@ -659,6 +659,9 @@
             "$ref": "#/definitions/v1BlockMeta"
           },
           "description": "List of the input blocks."
+        },
+        "status": {
+          "$ref": "#/definitions/v1CompactionJobStatus"
         }
       },
       "description": "One compaction job may result in multiple output blocks."
@@ -671,9 +674,6 @@
     "v1CompactionJobStatus": {
       "type": "object",
       "properties": {
-        "name": {
-          "type": "string"
-        },
         "status": {
           "$ref": "#/definitions/v1CompactionStatus",
           "description": "Status update allows the planner to keep\ntrack of the job ownership and compaction\nprogress:\n- If the job status is other than IN_PROGRESS,\n  the ownership of the job is revoked.\n- FAILURE must only be sent if the failure is\n  persistent and the compaction can't be accomplished.\n- completed_job must be empty if the status is\n  other than SUCCESS, and vice-versa.\n- UNSPECIFIED must be sent if the worker rejects\n  or cancels the compaction job.\n\nPartial results/status is not allowed."