feat: support headers for message (#1578)

Signed-off-by: Yashash H L <[email protected]> Signed-off-by: Vigith Maurice <[email protected]> Co-authored-by: Vigith Maurice <[email protected]>
numaproj · Mar 21, 2024 · 611bab7 · 611bab7
1 parent b394024
commit 611bab7
Show file tree

Hide file tree

Showing 45 changed files with 675 additions and 611 deletions.
diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json
@@ -18849,10 +18849,6 @@
         "consumerGroup": {
           "type": "string"
         },
-        "includeHeaders": {
-          "description": "include Kafka headers (this will change the payload struct into a JSON format with _body and _headers) EXPERIMENTAL: We might remove this option later on (https://github.com/numaproj/numaflow/issues/1568)",
-          "type": "boolean"
-        },
         "sasl": {
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.SASL",
           "description": "SASL user to configure SASL connection for kafka broker SASL.enable=true default for SASL."

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -18848,10 +18848,6 @@
         "consumerGroup": {
           "type": "string"
         },
-        "includeHeaders": {
-          "description": "include Kafka headers (this will change the payload struct into a JSON format with _body and _headers) EXPERIMENTAL: We might remove this option later on (https://github.com/numaproj/numaflow/issues/1568)",
-          "type": "boolean"
-        },
         "sasl": {
           "description": "SASL user to configure SASL connection for kafka broker SASL.enable=true default for SASL.",
           "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.SASL"

diff --git a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml
@@ -7323,8 +7323,6 @@ spec:
                               type: string
                             consumerGroup:
                               type: string
-                            includeHeaders:
-                              type: boolean
                             sasl:
                               properties:
                                 gssapi:

diff --git a/config/base/crds/full/numaflow.numaproj.io_vertices.yaml b/config/base/crds/full/numaflow.numaproj.io_vertices.yaml
@@ -2975,8 +2975,6 @@ spec:
                         type: string
                       consumerGroup:
                         type: string
-                      includeHeaders:
-                        type: boolean
                       sasl:
                         properties:
                           gssapi:

diff --git a/config/install.yaml b/config/install.yaml
@@ -9932,8 +9932,6 @@ spec:
                               type: string
                             consumerGroup:
                               type: string
-                            includeHeaders:
-                              type: boolean
                             sasl:
                               properties:
                                 gssapi:
@@ -14786,8 +14784,6 @@ spec:
                         type: string
                       consumerGroup:
                         type: string
-                      includeHeaders:
-                        type: boolean
                       sasl:
                         properties:
                           gssapi:

diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml
@@ -9932,8 +9932,6 @@ spec:
                               type: string
                             consumerGroup:
                               type: string
-                            includeHeaders:
-                              type: boolean
                             sasl:
                               properties:
                                 gssapi:
@@ -14786,8 +14784,6 @@ spec:
                         type: string
                       consumerGroup:
                         type: string
-                      includeHeaders:
-                        type: boolean
                       sasl:
                         properties:
                           gssapi:

diff --git a/docs/APIs.md b/docs/APIs.md
@@ -2784,20 +2784,6 @@ default for SASL.
 </p>
 </td>
 </tr>
-<tr>
-<td>
-<code>includeHeaders</code></br> <em> bool </em>
-</td>
-<td>
-<em>(Optional)</em>
-<p>
-include Kafka headers (this will change the payload struct into a JSON
-format with \_body and \_headers) EXPERIMENTAL: We might remove this
-option later on
-(<a href="https://github.com/numaproj/numaflow/issues/1568">https://github.com/numaproj/numaflow/issues/1568</a>)
-</p>
-</td>
-</tr>
 </tbody>
 </table>
 <h3 id="numaflow.numaproj.io/v1alpha1.Lifecycle">

diff --git a/go.mod b/go.mod
@@ -30,7 +30,7 @@ require (
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe
 	github.com/nats-io/nats-server/v2 v2.10.4
 	github.com/nats-io/nats.go v1.33.1
-	github.com/numaproj/numaflow-go v0.6.1-0.20240212202512-715955f1e069
+	github.com/numaproj/numaflow-go v0.6.1-0.20240319053512-a91b98f0bae8
 	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common v0.45.0
 	github.com/redis/go-redis/v9 v9.0.3

diff --git a/go.sum b/go.sum
@@ -669,8 +669,8 @@ github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDm
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/numaproj/numaflow-go v0.6.1-0.20240212202512-715955f1e069 h1:LBMIrUytoxuZazEnwxo+WPAxjHL1rrVhphTyByLiDYY=
-github.com/numaproj/numaflow-go v0.6.1-0.20240212202512-715955f1e069/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI=
+github.com/numaproj/numaflow-go v0.6.1-0.20240319053512-a91b98f0bae8 h1:bM6VZNEWHm9NCHsUqVk9dDdQIaa+rqqWakSWFXt0OM4=
+github.com/numaproj/numaflow-go v0.6.1-0.20240319053512-a91b98f0bae8/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=