From 0db9cd191ebb9406aa5dc94a0766597bc3f03f03 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Thu, 7 Dec 2023 08:45:39 +0530 Subject: [PATCH] feat: Generate Idle Watermark if the source is idling (#1385) Signed-off-by: Yashash H L Signed-off-by: chandankumar4 Signed-off-by: Vigith Maurice Co-authored-by: Yashash H L Co-authored-by: Vigith Maurice --- .github/workflows/ci.yaml | 4 +- api/json-schema/schema.json | 23 +- api/openapi-spec/swagger.json | 23 +- .../full/numaflow.numaproj.io_pipelines.yaml | 10 + .../full/numaflow.numaproj.io_vertices.yaml | 10 + config/install.yaml | 20 + config/namespace-install.yaml | 20 + docs/APIs.md | 85 +- pkg/apis/numaflow/v1alpha1/generated.pb.go | 1263 +++++++++++------ pkg/apis/numaflow/v1alpha1/generated.proto | 22 +- .../numaflow/v1alpha1/openapi_generated.go | 43 +- pkg/apis/numaflow/v1alpha1/pipeline_types.go | 42 +- .../v1alpha1/zz_generated.deepcopy.go | 36 + pkg/daemon/server/daemon_server.go | 2 +- .../server/service/pipeline_metrics_query.go | 4 +- .../service/pipeline_watermark_query.go | 6 +- pkg/isb/interfaces.go | 24 +- pkg/metrics/metrics_server.go | 6 +- pkg/reconciler/pipeline/validate.go | 23 + pkg/reconciler/pipeline/validate_test.go | 48 + pkg/shared/idlehandler/idlehandler.go | 1 - pkg/shared/idlehandler/source_idlehandler.go | 95 ++ .../idlehandler/source_idlehandler_test.go | 96 ++ pkg/sinks/sink.go | 8 +- pkg/sources/forward/data_forward.go | 271 ++-- pkg/sources/forward/data_forward_test.go | 73 +- pkg/sources/forward/shutdown.go | 34 +- pkg/sources/forward/shutdown_test.go | 8 +- pkg/sources/generator/tickgen.go | 132 +- pkg/sources/generator/tickgen_test.go | 12 +- pkg/sources/http/http.go | 78 +- pkg/sources/http/http_test.go | 12 +- pkg/sources/kafka/handler.go | 8 +- pkg/sources/kafka/handler_test.go | 4 +- pkg/sources/kafka/reader.go | 192 +-- pkg/sources/kafka/reader_test.go | 26 +- pkg/sources/nats/nats.go | 91 +- pkg/sources/nats/nats_test.go | 5 +- pkg/sources/source.go | 21 +- pkg/sources/sourcer.go | 31 - pkg/sources/sourcer/sourcer.go | 51 + pkg/sources/udsource/user_defined_source.go | 119 +- pkg/sources/udsource/utils/offset.go | 16 + pkg/sources/udsource/utils/offset_test.go | 16 + pkg/udf/forward/forward_test.go | 2 +- pkg/udf/map_udf.go | 9 +- pkg/udf/reduce_udf.go | 8 +- pkg/watermark/fetch/interface.go | 11 +- pkg/watermark/fetch/source_fetcher.go | 11 +- pkg/watermark/generic/noop.go | 55 + pkg/watermark/publish/options.go | 10 + pkg/watermark/publish/publisher.go | 10 +- pkg/watermark/publish/source_publisher.go | 123 ++ pkg/watermark/wmb/interface.go | 16 + pkg/watermark/wmb/noop_idle_manager.go | 16 + pkg/watermark/wmb/watermark.go | 4 + test/idle-source-e2e/idle_source_test.go | 75 + .../testdata/idle-source-reduce-pipeline.yaml | 55 + 58 files changed, 2392 insertions(+), 1127 deletions(-) create mode 100644 pkg/shared/idlehandler/source_idlehandler.go create mode 100644 pkg/shared/idlehandler/source_idlehandler_test.go delete mode 100644 pkg/sources/sourcer.go create mode 100644 pkg/sources/sourcer/sourcer.go create mode 100644 pkg/watermark/publish/source_publisher.go create mode 100644 test/idle-source-e2e/idle_source_test.go create mode 100644 test/idle-source-e2e/testdata/idle-source-reduce-pipeline.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 84a1082fb7..499a3167c9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -137,10 +137,10 @@ jobs: timeout-minutes: 20 strategy: fail-fast: false - max-parallel: 11 + max-parallel: 12 matrix: driver: [jetstream] - case: [e2e, diamond-e2e, transformer-e2e, kafka-e2e, http-e2e, nats-e2e, sdks-e2e, reduce-e2e, udsource-e2e, api-e2e, sideinputs-e2e] + case: [e2e, diamond-e2e, transformer-e2e, kafka-e2e, http-e2e, nats-e2e, sdks-e2e, reduce-e2e, udsource-e2e, api-e2e, sideinputs-e2e, idle-source-e2e] steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json index 186298a100..3db0f78308 100644 --- a/api/json-schema/schema.json +++ b/api/json-schema/schema.json @@ -18155,6 +18155,23 @@ }, "type": "object" }, + "io.numaproj.numaflow.v1alpha1.IdleSource": { + "properties": { + "incrementBy": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration", + "description": "IncrementBy is the duration to be added to the current watermark to progress the watermark when source is idling." + }, + "stepInterval": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration", + "description": "StepInterval is the duration between the subsequent increment of the watermark as long the source remains Idle. The default value is 0s which means that once we detect idle source, we will be incrementing the watermark by `IncrementBy` for time we detect that we source is empty (in other words, this will be a very frequent update)." + }, + "threshold": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration", + "description": "Threshold is the duration after which a source is marked as Idle due to lack of data. Ex: If watermark found to be idle after the Threshold duration then the watermark is progressed by `IncrementBy`." + } + }, + "type": "object" + }, "io.numaproj.numaflow.v1alpha1.InterStepBufferService": { "properties": { "apiVersion": { @@ -18858,7 +18875,7 @@ }, "templates": { "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Templates", - "description": "Templates is used to customize additional kubernetes resources required for the Pipeline" + "description": "Templates are used to customize additional kubernetes resources required for the Pipeline" }, "vertices": { "items": { @@ -19769,6 +19786,10 @@ "description": "Disabled toggles the watermark propagation, defaults to false.", "type": "boolean" }, + "idleSource": { + "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.IdleSource", + "description": "IdleSource defines the idle watermark properties, it could be configured in case source is idling." + }, "maxDelay": { "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration", "description": "Maximum delay allowed for watermark calculation, defaults to \"0s\", which means no delay." diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 0f6aceaeb7..7a5cfc2b61 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -18159,6 +18159,23 @@ } } }, + "io.numaproj.numaflow.v1alpha1.IdleSource": { + "type": "object", + "properties": { + "incrementBy": { + "description": "IncrementBy is the duration to be added to the current watermark to progress the watermark when source is idling.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + }, + "stepInterval": { + "description": "StepInterval is the duration between the subsequent increment of the watermark as long the source remains Idle. The default value is 0s which means that once we detect idle source, we will be incrementing the watermark by `IncrementBy` for time we detect that we source is empty (in other words, this will be a very frequent update).", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + }, + "threshold": { + "description": "Threshold is the duration after which a source is marked as Idle due to lack of data. Ex: If watermark found to be idle after the Threshold duration then the watermark is progressed by `IncrementBy`.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + } + } + }, "io.numaproj.numaflow.v1alpha1.InterStepBufferService": { "type": "object", "required": [ @@ -18844,7 +18861,7 @@ } }, "templates": { - "description": "Templates is used to customize additional kubernetes resources required for the Pipeline", + "description": "Templates are used to customize additional kubernetes resources required for the Pipeline", "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.Templates" }, "vertices": { @@ -19747,6 +19764,10 @@ "description": "Disabled toggles the watermark propagation, defaults to false.", "type": "boolean" }, + "idleSource": { + "description": "IdleSource defines the idle watermark properties, it could be configured in case source is idling.", + "$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.IdleSource" + }, "maxDelay": { "description": "Maximum delay allowed for watermark calculation, defaults to \"0s\", which means no delay.", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" diff --git a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml index 45c95d6e95..c2a91c1e02 100644 --- a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml +++ b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml @@ -8656,6 +8656,16 @@ spec: disabled: default: false type: boolean + idleSource: + properties: + incrementBy: + type: string + stepInterval: + default: 0s + type: string + threshold: + type: string + type: object maxDelay: default: 0s type: string diff --git a/config/base/crds/full/numaflow.numaproj.io_vertices.yaml b/config/base/crds/full/numaflow.numaproj.io_vertices.yaml index 54d2c4c018..11a85c29bb 100644 --- a/config/base/crds/full/numaflow.numaproj.io_vertices.yaml +++ b/config/base/crds/full/numaflow.numaproj.io_vertices.yaml @@ -4567,6 +4567,16 @@ spec: disabled: default: false type: boolean + idleSource: + properties: + incrementBy: + type: string + stepInterval: + default: 0s + type: string + threshold: + type: string + type: object maxDelay: default: 0s type: string diff --git a/config/install.yaml b/config/install.yaml index 3b448f0715..3354ba796b 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -11185,6 +11185,16 @@ spec: disabled: default: false type: boolean + idleSource: + properties: + incrementBy: + type: string + stepInterval: + default: 0s + type: string + threshold: + type: string + type: object maxDelay: default: 0s type: string @@ -15838,6 +15848,16 @@ spec: disabled: default: false type: boolean + idleSource: + properties: + incrementBy: + type: string + stepInterval: + default: 0s + type: string + threshold: + type: string + type: object maxDelay: default: 0s type: string diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml index c465d85a03..5972b3c64f 100644 --- a/config/namespace-install.yaml +++ b/config/namespace-install.yaml @@ -11185,6 +11185,16 @@ spec: disabled: default: false type: boolean + idleSource: + properties: + incrementBy: + type: string + stepInterval: + default: 0s + type: string + threshold: + type: string + type: object maxDelay: default: 0s type: string @@ -15838,6 +15848,16 @@ spec: disabled: default: false type: boolean + idleSource: + properties: + incrementBy: + type: string + stepInterval: + default: 0s + type: string + threshold: + type: string + type: object maxDelay: default: 0s type: string diff --git a/docs/APIs.md b/docs/APIs.md index 473ae4a664..bc750dafa4 100644 --- a/docs/APIs.md +++ b/docs/APIs.md @@ -1973,6 +1973,73 @@ ISBSvcType (string alias)

+

+IdleSource +

+

+(Appears on: +Watermark) +

+

+

+ + + + + + + + + + + + + + + + + + + + + +
+Field + +Description +
+threshold
+ +Kubernetes meta/v1.Duration +		 +

+Threshold is the duration after which a source is marked as Idle due to +lack of data. Ex: If watermark found to be idle after the Threshold +duration then the watermark is progressed by IncrementBy. +

+
+stepInterval
+ +Kubernetes meta/v1.Duration +		 +(Optional) +

+StepInterval is the duration between the subsequent increment of the +watermark as long the source remains Idle. The default value is 0s which +means that once we detect idle source, we will be incrementing the +watermark by IncrementBy for time we detect that we source +is empty (in other words, this will be a very frequent update). +

+
+incrementBy
+ +Kubernetes meta/v1.Duration +		 +

+IncrementBy is the duration to be added to the current watermark to +progress the watermark when source is idling. +

+

InterStepBufferService

@@ -3253,7 +3320,7 @@ Watermark enables watermark progression across the entire pipeline. (Optional)

-Templates is used to customize additional kubernetes resources required +Templates are used to customize additional kubernetes resources required for the Pipeline

@@ -3469,7 +3536,7 @@ Watermark enables watermark progression across the entire pipeline. (Optional)

-Templates is used to customize additional kubernetes resources required +Templates are used to customize additional kubernetes resources required for the Pipeline

@@ -5396,6 +5463,20 @@ means no delay.

+ + +idleSource
+ IdleSource + + + +(Optional) +

+IdleSource defines the idle watermark properties, it could be configured +in case source is idling. +

+ +

diff --git a/pkg/apis/numaflow/v1alpha1/generated.pb.go b/pkg/apis/numaflow/v1alpha1/generated.pb.go index 8677dc226c..bdff61ee03 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.pb.go +++ b/pkg/apis/numaflow/v1alpha1/generated.pb.go @@ -748,10 +748,38 @@ func (m *HTTPSource) XXX_DiscardUnknown() { var xxx_messageInfo_HTTPSource proto.InternalMessageInfo +func (m *IdleSource) Reset() { *m = IdleSource{} } +func (*IdleSource) ProtoMessage() {} +func (*IdleSource) Descriptor() ([]byte, []int) { + return fileDescriptor_9d0d1b17d3865563, []int{25} +} +func (m *IdleSource) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *IdleSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *IdleSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_IdleSource.Merge(m, src) +} +func (m *IdleSource) XXX_Size() int { + return m.Size() +} +func (m *IdleSource) XXX_DiscardUnknown() { + xxx_messageInfo_IdleSource.DiscardUnknown(m) +} + +var xxx_messageInfo_IdleSource proto.InternalMessageInfo + func (m *InterStepBufferService) Reset() { *m = InterStepBufferService{} } func (*InterStepBufferService) ProtoMessage() {} func (*InterStepBufferService) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{25} + return fileDescriptor_9d0d1b17d3865563, []int{26} } func (m *InterStepBufferService) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -779,7 +807,7 @@ var xxx_messageInfo_InterStepBufferService proto.InternalMessageInfo func (m *InterStepBufferServiceList) Reset() { *m = InterStepBufferServiceList{} } func (*InterStepBufferServiceList) ProtoMessage() {} func (*InterStepBufferServiceList) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{26} + return fileDescriptor_9d0d1b17d3865563, []int{27} } func (m *InterStepBufferServiceList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -807,7 +835,7 @@ var xxx_messageInfo_InterStepBufferServiceList proto.InternalMessageInfo func (m *InterStepBufferServiceSpec) Reset() { *m = InterStepBufferServiceSpec{} } func (*InterStepBufferServiceSpec) ProtoMessage() {} func (*InterStepBufferServiceSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{27} + return fileDescriptor_9d0d1b17d3865563, []int{28} } func (m *InterStepBufferServiceSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -835,7 +863,7 @@ var xxx_messageInfo_InterStepBufferServiceSpec proto.InternalMessageInfo func (m *InterStepBufferServiceStatus) Reset() { *m = InterStepBufferServiceStatus{} } func (*InterStepBufferServiceStatus) ProtoMessage() {} func (*InterStepBufferServiceStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{28} + return fileDescriptor_9d0d1b17d3865563, []int{29} } func (m *InterStepBufferServiceStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -863,7 +891,7 @@ var xxx_messageInfo_InterStepBufferServiceStatus proto.InternalMessageInfo func (m *JetStreamBufferService) Reset() { *m = JetStreamBufferService{} } func (*JetStreamBufferService) ProtoMessage() {} func (*JetStreamBufferService) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{29} + return fileDescriptor_9d0d1b17d3865563, []int{30} } func (m *JetStreamBufferService) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -891,7 +919,7 @@ var xxx_messageInfo_JetStreamBufferService proto.InternalMessageInfo func (m *JetStreamConfig) Reset() { *m = JetStreamConfig{} } func (*JetStreamConfig) ProtoMessage() {} func (*JetStreamConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{30} + return fileDescriptor_9d0d1b17d3865563, []int{31} } func (m *JetStreamConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -919,7 +947,7 @@ var xxx_messageInfo_JetStreamConfig proto.InternalMessageInfo func (m *JobTemplate) Reset() { *m = JobTemplate{} } func (*JobTemplate) ProtoMessage() {} func (*JobTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{31} + return fileDescriptor_9d0d1b17d3865563, []int{32} } func (m *JobTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -947,7 +975,7 @@ var xxx_messageInfo_JobTemplate proto.InternalMessageInfo func (m *KafkaSink) Reset() { *m = KafkaSink{} } func (*KafkaSink) ProtoMessage() {} func (*KafkaSink) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{32} + return fileDescriptor_9d0d1b17d3865563, []int{33} } func (m *KafkaSink) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -975,7 +1003,7 @@ var xxx_messageInfo_KafkaSink proto.InternalMessageInfo func (m *KafkaSource) Reset() { *m = KafkaSource{} } func (*KafkaSource) ProtoMessage() {} func (*KafkaSource) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{33} + return fileDescriptor_9d0d1b17d3865563, []int{34} } func (m *KafkaSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1003,7 +1031,7 @@ var xxx_messageInfo_KafkaSource proto.InternalMessageInfo func (m *Lifecycle) Reset() { *m = Lifecycle{} } func (*Lifecycle) ProtoMessage() {} func (*Lifecycle) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{34} + return fileDescriptor_9d0d1b17d3865563, []int{35} } func (m *Lifecycle) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1031,7 +1059,7 @@ var xxx_messageInfo_Lifecycle proto.InternalMessageInfo func (m *Log) Reset() { *m = Log{} } func (*Log) ProtoMessage() {} func (*Log) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{35} + return fileDescriptor_9d0d1b17d3865563, []int{36} } func (m *Log) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1059,7 +1087,7 @@ var xxx_messageInfo_Log proto.InternalMessageInfo func (m *Metadata) Reset() { *m = Metadata{} } func (*Metadata) ProtoMessage() {} func (*Metadata) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{36} + return fileDescriptor_9d0d1b17d3865563, []int{37} } func (m *Metadata) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1087,7 +1115,7 @@ var xxx_messageInfo_Metadata proto.InternalMessageInfo func (m *NativeRedis) Reset() { *m = NativeRedis{} } func (*NativeRedis) ProtoMessage() {} func (*NativeRedis) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{37} + return fileDescriptor_9d0d1b17d3865563, []int{38} } func (m *NativeRedis) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1115,7 +1143,7 @@ var xxx_messageInfo_NativeRedis proto.InternalMessageInfo func (m *NatsAuth) Reset() { *m = NatsAuth{} } func (*NatsAuth) ProtoMessage() {} func (*NatsAuth) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{38} + return fileDescriptor_9d0d1b17d3865563, []int{39} } func (m *NatsAuth) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1143,7 +1171,7 @@ var xxx_messageInfo_NatsAuth proto.InternalMessageInfo func (m *NatsSource) Reset() { *m = NatsSource{} } func (*NatsSource) ProtoMessage() {} func (*NatsSource) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{39} + return fileDescriptor_9d0d1b17d3865563, []int{40} } func (m *NatsSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1171,7 +1199,7 @@ var xxx_messageInfo_NatsSource proto.InternalMessageInfo func (m *PBQStorage) Reset() { *m = PBQStorage{} } func (*PBQStorage) ProtoMessage() {} func (*PBQStorage) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{40} + return fileDescriptor_9d0d1b17d3865563, []int{41} } func (m *PBQStorage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1199,7 +1227,7 @@ var xxx_messageInfo_PBQStorage proto.InternalMessageInfo func (m *PersistenceStrategy) Reset() { *m = PersistenceStrategy{} } func (*PersistenceStrategy) ProtoMessage() {} func (*PersistenceStrategy) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{41} + return fileDescriptor_9d0d1b17d3865563, []int{42} } func (m *PersistenceStrategy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1227,7 +1255,7 @@ var xxx_messageInfo_PersistenceStrategy proto.InternalMessageInfo func (m *Pipeline) Reset() { *m = Pipeline{} } func (*Pipeline) ProtoMessage() {} func (*Pipeline) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{42} + return fileDescriptor_9d0d1b17d3865563, []int{43} } func (m *Pipeline) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1255,7 +1283,7 @@ var xxx_messageInfo_Pipeline proto.InternalMessageInfo func (m *PipelineLimits) Reset() { *m = PipelineLimits{} } func (*PipelineLimits) ProtoMessage() {} func (*PipelineLimits) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{43} + return fileDescriptor_9d0d1b17d3865563, []int{44} } func (m *PipelineLimits) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1283,7 +1311,7 @@ var xxx_messageInfo_PipelineLimits proto.InternalMessageInfo func (m *PipelineList) Reset() { *m = PipelineList{} } func (*PipelineList) ProtoMessage() {} func (*PipelineList) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{44} + return fileDescriptor_9d0d1b17d3865563, []int{45} } func (m *PipelineList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1311,7 +1339,7 @@ var xxx_messageInfo_PipelineList proto.InternalMessageInfo func (m *PipelineSpec) Reset() { *m = PipelineSpec{} } func (*PipelineSpec) ProtoMessage() {} func (*PipelineSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{45} + return fileDescriptor_9d0d1b17d3865563, []int{46} } func (m *PipelineSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1339,7 +1367,7 @@ var xxx_messageInfo_PipelineSpec proto.InternalMessageInfo func (m *PipelineStatus) Reset() { *m = PipelineStatus{} } func (*PipelineStatus) ProtoMessage() {} func (*PipelineStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{46} + return fileDescriptor_9d0d1b17d3865563, []int{47} } func (m *PipelineStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1367,7 +1395,7 @@ var xxx_messageInfo_PipelineStatus proto.InternalMessageInfo func (m *RedisBufferService) Reset() { *m = RedisBufferService{} } func (*RedisBufferService) ProtoMessage() {} func (*RedisBufferService) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{47} + return fileDescriptor_9d0d1b17d3865563, []int{48} } func (m *RedisBufferService) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1395,7 +1423,7 @@ var xxx_messageInfo_RedisBufferService proto.InternalMessageInfo func (m *RedisConfig) Reset() { *m = RedisConfig{} } func (*RedisConfig) ProtoMessage() {} func (*RedisConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{48} + return fileDescriptor_9d0d1b17d3865563, []int{49} } func (m *RedisConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1423,7 +1451,7 @@ var xxx_messageInfo_RedisConfig proto.InternalMessageInfo func (m *RedisSettings) Reset() { *m = RedisSettings{} } func (*RedisSettings) ProtoMessage() {} func (*RedisSettings) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{49} + return fileDescriptor_9d0d1b17d3865563, []int{50} } func (m *RedisSettings) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1451,7 +1479,7 @@ var xxx_messageInfo_RedisSettings proto.InternalMessageInfo func (m *SASL) Reset() { *m = SASL{} } func (*SASL) ProtoMessage() {} func (*SASL) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{50} + return fileDescriptor_9d0d1b17d3865563, []int{51} } func (m *SASL) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1479,7 +1507,7 @@ var xxx_messageInfo_SASL proto.InternalMessageInfo func (m *SASLPlain) Reset() { *m = SASLPlain{} } func (*SASLPlain) ProtoMessage() {} func (*SASLPlain) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{51} + return fileDescriptor_9d0d1b17d3865563, []int{52} } func (m *SASLPlain) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1507,7 +1535,7 @@ var xxx_messageInfo_SASLPlain proto.InternalMessageInfo func (m *Scale) Reset() { *m = Scale{} } func (*Scale) ProtoMessage() {} func (*Scale) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{52} + return fileDescriptor_9d0d1b17d3865563, []int{53} } func (m *Scale) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1535,7 +1563,7 @@ var xxx_messageInfo_Scale proto.InternalMessageInfo func (m *SideInput) Reset() { *m = SideInput{} } func (*SideInput) ProtoMessage() {} func (*SideInput) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{53} + return fileDescriptor_9d0d1b17d3865563, []int{54} } func (m *SideInput) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1563,7 +1591,7 @@ var xxx_messageInfo_SideInput proto.InternalMessageInfo func (m *SideInputTrigger) Reset() { *m = SideInputTrigger{} } func (*SideInputTrigger) ProtoMessage() {} func (*SideInputTrigger) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{54} + return fileDescriptor_9d0d1b17d3865563, []int{55} } func (m *SideInputTrigger) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1591,7 +1619,7 @@ var xxx_messageInfo_SideInputTrigger proto.InternalMessageInfo func (m *SideInputsManagerTemplate) Reset() { *m = SideInputsManagerTemplate{} } func (*SideInputsManagerTemplate) ProtoMessage() {} func (*SideInputsManagerTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{55} + return fileDescriptor_9d0d1b17d3865563, []int{56} } func (m *SideInputsManagerTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1619,7 +1647,7 @@ var xxx_messageInfo_SideInputsManagerTemplate proto.InternalMessageInfo func (m *Sink) Reset() { *m = Sink{} } func (*Sink) ProtoMessage() {} func (*Sink) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{56} + return fileDescriptor_9d0d1b17d3865563, []int{57} } func (m *Sink) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1647,7 +1675,7 @@ var xxx_messageInfo_Sink proto.InternalMessageInfo func (m *SlidingWindow) Reset() { *m = SlidingWindow{} } func (*SlidingWindow) ProtoMessage() {} func (*SlidingWindow) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{57} + return fileDescriptor_9d0d1b17d3865563, []int{58} } func (m *SlidingWindow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1675,7 +1703,7 @@ var xxx_messageInfo_SlidingWindow proto.InternalMessageInfo func (m *Source) Reset() { *m = Source{} } func (*Source) ProtoMessage() {} func (*Source) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{58} + return fileDescriptor_9d0d1b17d3865563, []int{59} } func (m *Source) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1703,7 +1731,7 @@ var xxx_messageInfo_Source proto.InternalMessageInfo func (m *Status) Reset() { *m = Status{} } func (*Status) ProtoMessage() {} func (*Status) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{59} + return fileDescriptor_9d0d1b17d3865563, []int{60} } func (m *Status) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1731,7 +1759,7 @@ var xxx_messageInfo_Status proto.InternalMessageInfo func (m *TLS) Reset() { *m = TLS{} } func (*TLS) ProtoMessage() {} func (*TLS) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{60} + return fileDescriptor_9d0d1b17d3865563, []int{61} } func (m *TLS) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1759,7 +1787,7 @@ var xxx_messageInfo_TLS proto.InternalMessageInfo func (m *TagConditions) Reset() { *m = TagConditions{} } func (*TagConditions) ProtoMessage() {} func (*TagConditions) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{61} + return fileDescriptor_9d0d1b17d3865563, []int{62} } func (m *TagConditions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1787,7 +1815,7 @@ var xxx_messageInfo_TagConditions proto.InternalMessageInfo func (m *Templates) Reset() { *m = Templates{} } func (*Templates) ProtoMessage() {} func (*Templates) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{62} + return fileDescriptor_9d0d1b17d3865563, []int{63} } func (m *Templates) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1815,7 +1843,7 @@ var xxx_messageInfo_Templates proto.InternalMessageInfo func (m *Transformer) Reset() { *m = Transformer{} } func (*Transformer) ProtoMessage() {} func (*Transformer) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{63} + return fileDescriptor_9d0d1b17d3865563, []int{64} } func (m *Transformer) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1843,7 +1871,7 @@ var xxx_messageInfo_Transformer proto.InternalMessageInfo func (m *UDF) Reset() { *m = UDF{} } func (*UDF) ProtoMessage() {} func (*UDF) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{64} + return fileDescriptor_9d0d1b17d3865563, []int{65} } func (m *UDF) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1871,7 +1899,7 @@ var xxx_messageInfo_UDF proto.InternalMessageInfo func (m *UDSink) Reset() { *m = UDSink{} } func (*UDSink) ProtoMessage() {} func (*UDSink) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{65} + return fileDescriptor_9d0d1b17d3865563, []int{66} } func (m *UDSink) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1899,7 +1927,7 @@ var xxx_messageInfo_UDSink proto.InternalMessageInfo func (m *UDSource) Reset() { *m = UDSource{} } func (*UDSource) ProtoMessage() {} func (*UDSource) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{66} + return fileDescriptor_9d0d1b17d3865563, []int{67} } func (m *UDSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1927,7 +1955,7 @@ var xxx_messageInfo_UDSource proto.InternalMessageInfo func (m *UDTransformer) Reset() { *m = UDTransformer{} } func (*UDTransformer) ProtoMessage() {} func (*UDTransformer) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{67} + return fileDescriptor_9d0d1b17d3865563, []int{68} } func (m *UDTransformer) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1955,7 +1983,7 @@ var xxx_messageInfo_UDTransformer proto.InternalMessageInfo func (m *Vertex) Reset() { *m = Vertex{} } func (*Vertex) ProtoMessage() {} func (*Vertex) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{68} + return fileDescriptor_9d0d1b17d3865563, []int{69} } func (m *Vertex) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1983,7 +2011,7 @@ var xxx_messageInfo_Vertex proto.InternalMessageInfo func (m *VertexInstance) Reset() { *m = VertexInstance{} } func (*VertexInstance) ProtoMessage() {} func (*VertexInstance) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{69} + return fileDescriptor_9d0d1b17d3865563, []int{70} } func (m *VertexInstance) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2011,7 +2039,7 @@ var xxx_messageInfo_VertexInstance proto.InternalMessageInfo func (m *VertexLimits) Reset() { *m = VertexLimits{} } func (*VertexLimits) ProtoMessage() {} func (*VertexLimits) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{70} + return fileDescriptor_9d0d1b17d3865563, []int{71} } func (m *VertexLimits) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2039,7 +2067,7 @@ var xxx_messageInfo_VertexLimits proto.InternalMessageInfo func (m *VertexList) Reset() { *m = VertexList{} } func (*VertexList) ProtoMessage() {} func (*VertexList) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{71} + return fileDescriptor_9d0d1b17d3865563, []int{72} } func (m *VertexList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2067,7 +2095,7 @@ var xxx_messageInfo_VertexList proto.InternalMessageInfo func (m *VertexSpec) Reset() { *m = VertexSpec{} } func (*VertexSpec) ProtoMessage() {} func (*VertexSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{72} + return fileDescriptor_9d0d1b17d3865563, []int{73} } func (m *VertexSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2095,7 +2123,7 @@ var xxx_messageInfo_VertexSpec proto.InternalMessageInfo func (m *VertexStatus) Reset() { *m = VertexStatus{} } func (*VertexStatus) ProtoMessage() {} func (*VertexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{73} + return fileDescriptor_9d0d1b17d3865563, []int{74} } func (m *VertexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2123,7 +2151,7 @@ var xxx_messageInfo_VertexStatus proto.InternalMessageInfo func (m *VertexTemplate) Reset() { *m = VertexTemplate{} } func (*VertexTemplate) ProtoMessage() {} func (*VertexTemplate) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{74} + return fileDescriptor_9d0d1b17d3865563, []int{75} } func (m *VertexTemplate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2151,7 +2179,7 @@ var xxx_messageInfo_VertexTemplate proto.InternalMessageInfo func (m *Watermark) Reset() { *m = Watermark{} } func (*Watermark) ProtoMessage() {} func (*Watermark) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{75} + return fileDescriptor_9d0d1b17d3865563, []int{76} } func (m *Watermark) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2179,7 +2207,7 @@ var xxx_messageInfo_Watermark proto.InternalMessageInfo func (m *Window) Reset() { *m = Window{} } func (*Window) ProtoMessage() {} func (*Window) Descriptor() ([]byte, []int) { - return fileDescriptor_9d0d1b17d3865563, []int{76} + return fileDescriptor_9d0d1b17d3865563, []int{77} } func (m *Window) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2236,6 +2264,7 @@ func init() { proto.RegisterType((*GetVertexPodSpecReq)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.GetVertexPodSpecReq") proto.RegisterType((*GroupBy)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.GroupBy") proto.RegisterType((*HTTPSource)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.HTTPSource") + proto.RegisterType((*IdleSource)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.IdleSource") proto.RegisterType((*InterStepBufferService)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.InterStepBufferService") proto.RegisterType((*InterStepBufferServiceList)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.InterStepBufferServiceList") proto.RegisterType((*InterStepBufferServiceSpec)(nil), "github.com.numaproj.numaflow.pkg.apis.numaflow.v1alpha1.InterStepBufferServiceSpec") @@ -2298,417 +2327,422 @@ func init() { } var fileDescriptor_9d0d1b17d3865563 = []byte{ - // 6555 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x5b, 0x6c, 0x24, 0xd9, - 0x59, 0xf0, 0xf6, 0xcd, 0xee, 0xfe, 0xda, 0xf6, 0xcc, 0x9c, 0xb9, 0xac, 0xc7, 0x3b, 0x3b, 0x3d, - 0xa9, 0xfc, 0xbb, 0xff, 0xe4, 0xff, 0x13, 0x0f, 0x6b, 0x36, 0xec, 0x06, 0x48, 0x76, 0xdd, 0xf6, - 0xd8, 0x3b, 0x3b, 0xf6, 0x8c, 0xf3, 0xb5, 0x3d, 0xb3, 0xc9, 0x92, 0x2c, 0xe5, 0xea, 0xe3, 0x76, - 0x6d, 0x57, 0x57, 0x75, 0xaa, 0x4e, 0x7b, 0xc6, 0x1b, 0x22, 0x12, 0xf2, 0xb0, 0x89, 0x00, 0x05, - 0x81, 0x84, 0x22, 0x50, 0x40, 0x48, 0x48, 0x3c, 0xa0, 0xbc, 0x11, 0x1e, 0x40, 0x08, 0x78, 0x41, - 0x81, 0x07, 0xc8, 0x03, 0x52, 0x82, 0x82, 0x5a, 0xc4, 0x3c, 0x21, 0x44, 0x14, 0x11, 0x09, 0x45, - 0x23, 0x24, 0xd0, 0xb9, 0xd5, 0xad, 0xab, 0x67, 0xec, 0x6e, 0x7b, 0x32, 0x81, 0xbc, 0x75, 0x9d, - 0xef, 0x76, 0xea, 0xd4, 0x39, 0xdf, 0xf9, 0x6e, 0xe7, 0x34, 0xac, 0xb6, 0x6c, 0xb6, 0xdb, 0xdb, - 0x9e, 0xb7, 0xbc, 0xce, 0x35, 0xb7, 0xd7, 0x31, 0xbb, 0xbe, 0xf7, 0xb6, 0xf8, 0xb1, 0xe3, 0x78, - 0xf7, 0xae, 0x75, 0xdb, 0xad, 0x6b, 0x66, 0xd7, 0x0e, 0xa2, 0x96, 0xbd, 0x17, 0x4c, 0xa7, 0xbb, - 0x6b, 0xbe, 0x70, 0xad, 0x45, 0x5d, 0xea, 0x9b, 0x8c, 0x36, 0xe7, 0xbb, 0xbe, 0xc7, 0x3c, 0xf2, - 0x52, 0xc4, 0x68, 0x5e, 0x33, 0x9a, 0xd7, 0x64, 0xf3, 0xdd, 0x76, 0x6b, 0x9e, 0x33, 0x8a, 0x5a, - 0x34, 0xa3, 0xb9, 0x0f, 0xc4, 0x7a, 0xd0, 0xf2, 0x5a, 0xde, 0x35, 0xc1, 0x6f, 0xbb, 0xb7, 0x23, - 0x9e, 0xc4, 0x83, 0xf8, 0x25, 0xe5, 0xcc, 0x19, 0xed, 0x97, 0x83, 0x79, 0xdb, 0xe3, 0xdd, 0xba, - 0x66, 0x79, 0x3e, 0xbd, 0xb6, 0x37, 0xd0, 0x97, 0xb9, 0x17, 0x23, 0x9c, 0x8e, 0x69, 0xed, 0xda, - 0x2e, 0xf5, 0xf7, 0xf5, 0xbb, 0x5c, 0xf3, 0x69, 0xe0, 0xf5, 0x7c, 0x8b, 0x1e, 0x89, 0x2a, 0xb8, - 0xd6, 0xa1, 0xcc, 0xcc, 0x92, 0x75, 0x6d, 0x18, 0x95, 0xdf, 0x73, 0x99, 0xdd, 0x19, 0x14, 0xf3, - 0x53, 0x8f, 0x22, 0x08, 0xac, 0x5d, 0xda, 0x31, 0xd3, 0x74, 0xc6, 0xb7, 0x2b, 0x70, 0x76, 0x71, - 0x3b, 0x60, 0xbe, 0x69, 0xb1, 0x0d, 0xaf, 0xb9, 0x49, 0x3b, 0x5d, 0xc7, 0x64, 0x94, 0xb4, 0xa1, - 0xcc, 0xfb, 0xd6, 0x34, 0x99, 0x39, 0x9b, 0xbb, 0x92, 0xbb, 0x5a, 0x5d, 0x58, 0x9c, 0x1f, 0xf1, - 0x5b, 0xcc, 0xaf, 0x2b, 0x46, 0xf5, 0xa9, 0x83, 0x7e, 0xad, 0xac, 0x9f, 0x30, 0x14, 0x40, 0xbe, - 0x9c, 0x83, 0x29, 0xd7, 0x6b, 0xd2, 0x06, 0x75, 0xa8, 0xc5, 0x3c, 0x7f, 0x36, 0x7f, 0xa5, 0x70, - 0xb5, 0xba, 0xf0, 0xc9, 0x91, 0x25, 0x66, 0xbc, 0xd1, 0xfc, 0xad, 0x98, 0x80, 0xeb, 0x2e, 0xf3, - 0xf7, 0xeb, 0xe7, 0xbe, 0xde, 0xaf, 0x3d, 0x75, 0xd0, 0xaf, 0x4d, 0xc5, 0x41, 0x98, 0xe8, 0x09, - 0xd9, 0x82, 0x2a, 0xf3, 0x1c, 0x3e, 0x64, 0xb6, 0xe7, 0x06, 0xb3, 0x05, 0xd1, 0xb1, 0xcb, 0xf3, - 0x72, 0xb4, 0xb9, 0xf8, 0x79, 0x3e, 0x5d, 0xe6, 0xf7, 0x5e, 0x98, 0xdf, 0x0c, 0xd1, 0xea, 0x67, - 0x15, 0xe3, 0x6a, 0xd4, 0x16, 0x60, 0x9c, 0x0f, 0xa1, 0x70, 0x2a, 0xa0, 0x56, 0xcf, 0xb7, 0xd9, - 0xfe, 0x92, 0xe7, 0x32, 0x7a, 0x9f, 0xcd, 0x16, 0xc5, 0x28, 0x3f, 0x9f, 0xc5, 0x7a, 0xc3, 0x6b, - 0x36, 0x92, 0xd8, 0xf5, 0xb3, 0x07, 0xfd, 0xda, 0xa9, 0x54, 0x23, 0xa6, 0x79, 0x12, 0x17, 0x4e, - 0xdb, 0x1d, 0xb3, 0x45, 0x37, 0x7a, 0x8e, 0xd3, 0xa0, 0x96, 0x4f, 0x59, 0x30, 0x5b, 0x12, 0xaf, - 0x70, 0x35, 0x4b, 0xce, 0x9a, 0x67, 0x99, 0xce, 0xed, 0xed, 0xb7, 0xa9, 0xc5, 0x90, 0xee, 0x50, - 0x9f, 0xba, 0x16, 0xad, 0xcf, 0xaa, 0x97, 0x39, 0x7d, 0x23, 0xc5, 0x09, 0x07, 0x78, 0x93, 0x55, - 0x38, 0xd3, 0xf5, 0x6d, 0x4f, 0x74, 0xc1, 0x31, 0x83, 0xe0, 0x96, 0xd9, 0xa1, 0xb3, 0x13, 0x57, - 0x72, 0x57, 0x2b, 0xf5, 0x8b, 0x8a, 0xcd, 0x99, 0x8d, 0x34, 0x02, 0x0e, 0xd2, 0x90, 0xab, 0x50, - 0xd6, 0x8d, 0xb3, 0x93, 0x57, 0x72, 0x57, 0x4b, 0x72, 0xee, 0x68, 0x5a, 0x0c, 0xa1, 0x64, 0x05, - 0xca, 0xe6, 0xce, 0x8e, 0xed, 0x72, 0xcc, 0xb2, 0x18, 0xc2, 0x4b, 0x59, 0xaf, 0xb6, 0xa8, 0x70, - 0x24, 0x1f, 0xfd, 0x84, 0x21, 0x2d, 0x79, 0x1d, 0x48, 0x40, 0xfd, 0x3d, 0xdb, 0xa2, 0x8b, 0x96, - 0xe5, 0xf5, 0x5c, 0x26, 0xfa, 0x5e, 0x11, 0x7d, 0x9f, 0x53, 0x7d, 0x27, 0x8d, 0x01, 0x0c, 0xcc, - 0xa0, 0x22, 0xaf, 0xc2, 0x69, 0xb5, 0xec, 0xa2, 0x51, 0x00, 0xc1, 0xe9, 0x1c, 0x1f, 0x48, 0x4c, - 0xc1, 0x70, 0x00, 0x9b, 0x34, 0xe1, 0x92, 0xd9, 0x63, 0x5e, 0x87, 0xb3, 0x4c, 0x0a, 0xdd, 0xf4, - 0xda, 0xd4, 0x9d, 0xad, 0x5e, 0xc9, 0x5d, 0x2d, 0xd7, 0xaf, 0x1c, 0xf4, 0x6b, 0x97, 0x16, 0x1f, - 0x82, 0x87, 0x0f, 0xe5, 0x42, 0x6e, 0x43, 0xa5, 0xe9, 0x06, 0x1b, 0x9e, 0x63, 0x5b, 0xfb, 0xb3, - 0x53, 0xa2, 0x83, 0x2f, 0xa8, 0x57, 0xad, 0x2c, 0xdf, 0x6a, 0x48, 0xc0, 0x83, 0x7e, 0xed, 0xd2, - 0xa0, 0x76, 0x9c, 0x0f, 0xe1, 0x18, 0xf1, 0x20, 0xeb, 0x82, 0xe1, 0x92, 0xe7, 0xee, 0xd8, 0xad, - 0xd9, 0x69, 0xf1, 0x35, 0xae, 0x0c, 0x99, 0xd0, 0xcb, 0xb7, 0x1a, 0x12, 0xaf, 0x3e, 0xad, 0xc4, - 0xc9, 0x47, 0x8c, 0x38, 0xcc, 0xbd, 0x02, 0x67, 0x06, 0x56, 0x2d, 0x39, 0x0d, 0x85, 0x36, 0xdd, - 0x17, 0x4a, 0xa9, 0x82, 0xfc, 0x27, 0x39, 0x07, 0xa5, 0x3d, 0xd3, 0xe9, 0xd1, 0xd9, 0xbc, 0x68, - 0x93, 0x0f, 0x3f, 0x9d, 0x7f, 0x39, 0x67, 0x7c, 0xb7, 0x0a, 0x33, 0x5a, 0x17, 0xdc, 0xa1, 0x3e, - 0xa3, 0xf7, 0xc9, 0x15, 0x28, 0xba, 0xfc, 0x7b, 0x08, 0xfa, 0xfa, 0x94, 0x7a, 0xdd, 0xa2, 0xf8, - 0x0e, 0x02, 0x42, 0x2c, 0x98, 0x90, 0xba, 0x5c, 0xf0, 0xab, 0x2e, 0xbc, 0x32, 0xb2, 0x1a, 0x6a, - 0x08, 0x36, 0x75, 0x38, 0xe8, 0xd7, 0x26, 0xe4, 0x6f, 0x54, 0xac, 0xc9, 0x9b, 0x50, 0x0c, 0x6c, - 0xb7, 0x3d, 0x5b, 0x10, 0x22, 0x3e, 0x3c, 0xba, 0x08, 0xdb, 0x6d, 0xd7, 0xcb, 0xfc, 0x0d, 0xf8, - 0x2f, 0x14, 0x4c, 0xc9, 0x5d, 0x28, 0xf4, 0x9a, 0x3b, 0x4a, 0xa3, 0xfc, 0xec, 0xc8, 0xbc, 0xb7, - 0x96, 0x57, 0xea, 0x93, 0x07, 0xfd, 0x5a, 0x61, 0x6b, 0x79, 0x05, 0x39, 0x47, 0xf2, 0xa5, 0x1c, - 0x9c, 0xb1, 0x3c, 0x97, 0x99, 0x7c, 0x7f, 0xd1, 0x9a, 0x75, 0xb6, 0x24, 0xe4, 0xbc, 0x3e, 0xb2, - 0x9c, 0xa5, 0x34, 0xc7, 0xfa, 0x79, 0xae, 0x28, 0x06, 0x9a, 0x71, 0x50, 0x36, 0xf9, 0xed, 0x1c, - 0x9c, 0xe7, 0x0b, 0x78, 0x00, 0x59, 0xa8, 0x9d, 0xe3, 0xed, 0xd5, 0xc5, 0x83, 0x7e, 0xed, 0xfc, - 0x8d, 0x2c, 0x61, 0x98, 0xdd, 0x07, 0xde, 0xbb, 0xb3, 0xe6, 0xe0, 0x5e, 0x24, 0x54, 0x5a, 0x75, - 0x61, 0xed, 0x38, 0xf7, 0xb7, 0xfa, 0x33, 0x6a, 0x2a, 0x67, 0x6d, 0xe7, 0x98, 0xd5, 0x0b, 0x72, - 0x1d, 0x26, 0xf7, 0x3c, 0xa7, 0xd7, 0xa1, 0xc1, 0x6c, 0x59, 0x6c, 0x0a, 0x73, 0x59, 0x6b, 0xf5, - 0x8e, 0x40, 0xa9, 0x9f, 0x52, 0xec, 0x27, 0xe5, 0x73, 0x80, 0x9a, 0x96, 0xd8, 0x30, 0xe1, 0xd8, - 0x1d, 0x9b, 0x05, 0x42, 0x5b, 0x56, 0x17, 0xae, 0x8f, 0xfc, 0x5a, 0x72, 0x89, 0xae, 0x09, 0x66, - 0x72, 0xd5, 0xc8, 0xdf, 0xa8, 0x04, 0x10, 0x0b, 0x4a, 0x81, 0x65, 0x3a, 0x52, 0x9b, 0x56, 0x17, - 0x3e, 0x32, 0xfa, 0xb2, 0xe1, 0x5c, 0xea, 0xd3, 0xea, 0x9d, 0x4a, 0xe2, 0x11, 0x25, 0x6f, 0xf2, - 0x09, 0x98, 0x49, 0x7c, 0xcd, 0x60, 0xb6, 0x2a, 0x46, 0xe7, 0xd9, 0xac, 0xd1, 0x09, 0xb1, 0xea, - 0x17, 0x14, 0xb3, 0x99, 0xc4, 0x0c, 0x09, 0x30, 0xc5, 0x8c, 0xdc, 0x84, 0x72, 0x60, 0x37, 0xa9, - 0x65, 0xfa, 0xc1, 0xec, 0xd4, 0x61, 0x18, 0x9f, 0x56, 0x8c, 0xcb, 0x0d, 0x45, 0x86, 0x21, 0x03, - 0x32, 0x0f, 0xd0, 0x35, 0x7d, 0x66, 0x4b, 0xeb, 0x64, 0x5a, 0xec, 0x94, 0x33, 0x07, 0xfd, 0x1a, - 0x6c, 0x84, 0xad, 0x18, 0xc3, 0xe0, 0xf8, 0x9c, 0xf6, 0x86, 0xdb, 0xed, 0xb1, 0x60, 0x76, 0xe6, - 0x4a, 0xe1, 0x6a, 0x45, 0xe2, 0x37, 0xc2, 0x56, 0x8c, 0x61, 0x90, 0xaf, 0xe6, 0xe0, 0x99, 0xe8, - 0x71, 0x70, 0x91, 0x9d, 0x3a, 0xf6, 0x45, 0x56, 0x3b, 0xe8, 0xd7, 0x9e, 0x69, 0x0c, 0x17, 0x89, - 0x0f, 0xeb, 0x8f, 0x71, 0x17, 0xa6, 0x17, 0x7b, 0x6c, 0xd7, 0xf3, 0xed, 0x77, 0x84, 0xa5, 0x45, - 0x56, 0xa0, 0xc4, 0xc4, 0x8e, 0x29, 0x8d, 0xd8, 0xe7, 0xb2, 0x86, 0x5a, 0x5a, 0x2f, 0x37, 0xe9, - 0xbe, 0xde, 0x68, 0xea, 0x15, 0x3e, 0x29, 0xe4, 0x0e, 0x2a, 0xc9, 0x8d, 0xdf, 0xcb, 0x41, 0xa5, - 0x6e, 0x06, 0xb6, 0xc5, 0xd9, 0x93, 0x25, 0x28, 0xf6, 0x02, 0xea, 0x1f, 0x8d, 0xa9, 0xd0, 0xd2, - 0x5b, 0x01, 0xf5, 0x51, 0x10, 0x93, 0xdb, 0x50, 0xee, 0x9a, 0x41, 0x70, 0xcf, 0xf3, 0x9b, 0x6a, - 0xa7, 0x39, 0x24, 0x23, 0x69, 0x0a, 0x29, 0x52, 0x0c, 0x99, 0x18, 0x55, 0xa8, 0xd4, 0x1d, 0xd3, - 0x6a, 0xef, 0x7a, 0x0e, 0x35, 0xbe, 0x9f, 0x83, 0xb3, 0xf5, 0xde, 0xce, 0x0e, 0xf5, 0xd5, 0xce, - 0x2f, 0xf7, 0x54, 0x42, 0xa1, 0xe4, 0xd3, 0xa6, 0x1d, 0xa8, 0xbe, 0x2f, 0x8f, 0xfc, 0xe9, 0x90, - 0x73, 0x51, 0x5b, 0xb8, 0x18, 0x2f, 0xd1, 0x80, 0x92, 0x3b, 0xe9, 0x41, 0xe5, 0x6d, 0xca, 0x02, - 0xe6, 0x53, 0xb3, 0xa3, 0xde, 0xee, 0xb5, 0x91, 0x45, 0xbd, 0x4e, 0x59, 0x43, 0x70, 0x8a, 0x5b, - 0x0c, 0x61, 0x23, 0x46, 0x92, 0x8c, 0xbf, 0x2c, 0xc1, 0xd4, 0x92, 0xd7, 0xd9, 0xb6, 0x5d, 0xda, - 0xbc, 0xde, 0x6c, 0x51, 0xf2, 0x16, 0x14, 0x69, 0xb3, 0x45, 0xd5, 0xdb, 0x8e, 0xbe, 0xcf, 0x72, - 0x66, 0x91, 0xb5, 0xc0, 0x9f, 0x50, 0x30, 0x26, 0x6b, 0x30, 0xb3, 0xe3, 0x7b, 0x1d, 0xa9, 0xba, - 0x36, 0xf7, 0xbb, 0xca, 0x0a, 0xa9, 0xff, 0x1f, 0xad, 0x0e, 0x56, 0x12, 0xd0, 0x07, 0xfd, 0x1a, - 0x44, 0x4f, 0x98, 0xa2, 0x25, 0x6f, 0xc0, 0x6c, 0xd4, 0x12, 0xae, 0xe1, 0x25, 0x6e, 0xb2, 0x09, - 0x53, 0xa1, 0x54, 0xbf, 0x74, 0xd0, 0xaf, 0xcd, 0xae, 0x0c, 0xc1, 0xc1, 0xa1, 0xd4, 0xe4, 0xdd, - 0x1c, 0x9c, 0x8e, 0x80, 0x52, 0xaf, 0x2a, 0x0b, 0xe1, 0x98, 0x14, 0xb6, 0xb0, 0x6d, 0x57, 0x52, - 0x22, 0x70, 0x40, 0x28, 0x59, 0x81, 0x29, 0xe6, 0xc5, 0xc6, 0xab, 0x24, 0xc6, 0xcb, 0xd0, 0xce, - 0xd8, 0xa6, 0x37, 0x74, 0xb4, 0x12, 0x74, 0x04, 0xe1, 0x82, 0x7e, 0x4e, 0x8d, 0xd4, 0x84, 0x18, - 0xa9, 0xb9, 0x83, 0x7e, 0xed, 0xc2, 0x66, 0x26, 0x06, 0x0e, 0xa1, 0x24, 0x9f, 0xcb, 0xc1, 0x8c, - 0x06, 0xa9, 0x31, 0x9a, 0x3c, 0xce, 0x31, 0x22, 0x7c, 0x46, 0x6c, 0x26, 0x04, 0x60, 0x4a, 0xa0, - 0xf1, 0x83, 0x22, 0x54, 0x42, 0xcd, 0x46, 0xde, 0x0b, 0x25, 0xe1, 0x66, 0x29, 0x83, 0x35, 0xdc, - 0xb2, 0x84, 0x37, 0x86, 0x12, 0x46, 0x9e, 0x83, 0x49, 0xcb, 0xeb, 0x74, 0x4c, 0xb7, 0x29, 0x5c, - 0xe7, 0x4a, 0xbd, 0xca, 0x77, 0xea, 0x25, 0xd9, 0x84, 0x1a, 0x46, 0x2e, 0x41, 0xd1, 0xf4, 0x5b, - 0xd2, 0x8b, 0xad, 0x48, 0x7d, 0xb4, 0xe8, 0xb7, 0x02, 0x14, 0xad, 0xe4, 0x43, 0x50, 0xa0, 0xee, - 0xde, 0x6c, 0x71, 0xb8, 0x29, 0x70, 0xdd, 0xdd, 0xbb, 0x63, 0xfa, 0xf5, 0xaa, 0xea, 0x43, 0xe1, - 0xba, 0xbb, 0x87, 0x9c, 0x86, 0xac, 0xc1, 0x24, 0x75, 0xf7, 0xf8, 0xb7, 0x57, 0xee, 0xe5, 0x7b, - 0x86, 0x90, 0x73, 0x14, 0x65, 0x15, 0x87, 0x06, 0x85, 0x6a, 0x46, 0xcd, 0x82, 0x7c, 0x0c, 0xa6, - 0xa4, 0x6d, 0xb1, 0xce, 0xbf, 0x49, 0x30, 0x3b, 0x21, 0x58, 0xd6, 0x86, 0x1b, 0x27, 0x02, 0x2f, - 0x72, 0xe7, 0x63, 0x8d, 0x01, 0x26, 0x58, 0x91, 0x8f, 0x41, 0x45, 0x47, 0x6a, 0xf4, 0x97, 0xcd, - 0xf4, 0x84, 0x51, 0x21, 0x21, 0xfd, 0x54, 0xcf, 0xf6, 0x69, 0x87, 0xba, 0x2c, 0xa8, 0x9f, 0xd1, - 0xbe, 0x91, 0x86, 0x06, 0x18, 0x71, 0x23, 0xdb, 0x83, 0x2e, 0xbd, 0xf4, 0x47, 0xdf, 0x3b, 0x44, - 0xab, 0x8f, 0xe0, 0xcf, 0x7f, 0x12, 0x4e, 0x85, 0x3e, 0xb7, 0x72, 0xdb, 0xa4, 0x87, 0xfa, 0x22, - 0x27, 0xbf, 0x91, 0x04, 0x3d, 0xe8, 0xd7, 0x9e, 0xcd, 0x70, 0xdc, 0x22, 0x04, 0x4c, 0x33, 0x33, - 0xfe, 0xbc, 0x00, 0x83, 0x66, 0x77, 0x72, 0xd0, 0x72, 0xc7, 0x3d, 0x68, 0xe9, 0x17, 0x92, 0xea, - 0xf3, 0x65, 0x45, 0x36, 0xfe, 0x4b, 0x65, 0x7d, 0x98, 0xc2, 0x71, 0x7f, 0x98, 0x27, 0x65, 0xed, - 0x18, 0x5f, 0x28, 0xc2, 0xcc, 0xb2, 0x49, 0x3b, 0x9e, 0xfb, 0x48, 0x27, 0x24, 0xf7, 0x44, 0x38, - 0x21, 0x57, 0xa1, 0xec, 0xd3, 0xae, 0x63, 0x5b, 0x66, 0x20, 0x3e, 0xbd, 0x8a, 0xf4, 0xa0, 0x6a, - 0xc3, 0x10, 0x3a, 0xc4, 0xf9, 0x2c, 0x3c, 0x91, 0xce, 0x67, 0xf1, 0x87, 0xef, 0x7c, 0x1a, 0x9f, - 0xcb, 0x83, 0x30, 0x54, 0xc8, 0x15, 0x28, 0xf2, 0x4d, 0x38, 0x1d, 0xf2, 0x10, 0x13, 0x47, 0x40, - 0xc8, 0x1c, 0xe4, 0x99, 0xa7, 0x56, 0x1e, 0x28, 0x78, 0x7e, 0xd3, 0xc3, 0x3c, 0xf3, 0xc8, 0x3b, - 0x00, 0x96, 0xe7, 0x36, 0x6d, 0x1d, 0x00, 0x1d, 0xef, 0xc5, 0x56, 0x3c, 0xff, 0x9e, 0xe9, 0x37, - 0x97, 0x42, 0x8e, 0xd2, 0xfd, 0x88, 0x9e, 0x31, 0x26, 0x8d, 0xbc, 0x02, 0x13, 0x9e, 0xbb, 0xd2, - 0x73, 0x1c, 0x31, 0xa0, 0x95, 0xfa, 0xff, 0xe5, 0x3e, 0xe1, 0x6d, 0xd1, 0xf2, 0xa0, 0x5f, 0xbb, - 0x28, 0xed, 0x5b, 0xfe, 0x74, 0xd7, 0xb7, 0x99, 0xed, 0xb6, 0x1a, 0xcc, 0x37, 0x19, 0x6d, 0xed, - 0xa3, 0x22, 0x33, 0x4c, 0xa8, 0xae, 0xd8, 0xf7, 0x69, 0xf3, 0xae, 0xed, 0x36, 0xbd, 0x7b, 0x04, - 0x61, 0xc2, 0xa1, 0x6e, 0x8b, 0xed, 0xaa, 0xc9, 0x3f, 0x1f, 0x5b, 0x6a, 0x61, 0xd8, 0x3c, 0xea, - 0x7e, 0x87, 0x32, 0x93, 0x2f, 0xbe, 0xe5, 0x9e, 0x0a, 0xec, 0x4a, 0x9f, 0x54, 0x70, 0x40, 0xc5, - 0xc9, 0xd8, 0x87, 0x33, 0x03, 0x2f, 0x45, 0x9a, 0x50, 0x64, 0x66, 0x4b, 0x6b, 0xcb, 0x95, 0x91, - 0x87, 0x6b, 0xd3, 0x6c, 0xc5, 0x86, 0x4a, 0xec, 0xd8, 0x9b, 0x26, 0xdf, 0xb1, 0x39, 0x77, 0xe3, - 0x3f, 0x73, 0x50, 0x5e, 0xe9, 0xb9, 0x96, 0xf0, 0x74, 0x1e, 0x1d, 0xd8, 0xd2, 0xdb, 0x7f, 0x3e, - 0x73, 0xfb, 0xef, 0xc1, 0x44, 0xfb, 0x5e, 0x68, 0x1e, 0x54, 0x17, 0xd6, 0x47, 0xff, 0xc6, 0xaa, - 0x4b, 0xf3, 0x37, 0x05, 0x3f, 0x19, 0x6c, 0x9f, 0x51, 0x1d, 0x9a, 0xb8, 0x79, 0x57, 0x08, 0x55, - 0xc2, 0xe6, 0x3e, 0x04, 0xd5, 0x18, 0xda, 0x91, 0xa2, 0x7b, 0x7f, 0x5c, 0x84, 0x89, 0xd5, 0x46, - 0x63, 0x71, 0xe3, 0x06, 0xf9, 0x20, 0x54, 0x55, 0x1c, 0xf6, 0x56, 0x34, 0x06, 0x61, 0x18, 0xbe, - 0x11, 0x81, 0x30, 0x8e, 0xc7, 0x8d, 0x2b, 0x9f, 0x9a, 0x4e, 0x47, 0x4d, 0xfd, 0xd0, 0xb8, 0x42, - 0xde, 0x88, 0x12, 0x46, 0x4c, 0x98, 0xe1, 0xfe, 0x1a, 0x1f, 0x42, 0xe9, 0x8b, 0xa9, 0x45, 0x70, - 0x48, 0x6f, 0x4d, 0x98, 0x7c, 0x5b, 0x09, 0x06, 0x98, 0x62, 0x48, 0x5e, 0x86, 0xb2, 0xd9, 0x63, - 0xbb, 0xc2, 0x1c, 0x96, 0x33, 0xfd, 0x92, 0x08, 0x53, 0xab, 0xb6, 0x07, 0xfd, 0xda, 0xd4, 0x4d, - 0xac, 0x7f, 0x50, 0x3f, 0x63, 0x88, 0xcd, 0x3b, 0xa7, 0xfd, 0x3f, 0xd5, 0xb9, 0xd2, 0x91, 0x3b, - 0xb7, 0x91, 0x60, 0x80, 0x29, 0x86, 0xe4, 0x4d, 0x98, 0x6a, 0xd3, 0x7d, 0x66, 0x6e, 0x2b, 0x01, - 0x13, 0x47, 0x11, 0x70, 0x9a, 0x1b, 0x64, 0x37, 0x63, 0xe4, 0x98, 0x60, 0x46, 0x02, 0x38, 0xd7, - 0xa6, 0xfe, 0x36, 0xf5, 0x3d, 0xe5, 0x4b, 0x2a, 0x21, 0x93, 0x47, 0x11, 0x32, 0x7b, 0xd0, 0xaf, - 0x9d, 0xbb, 0x99, 0xc1, 0x06, 0x33, 0x99, 0x1b, 0x3f, 0xc8, 0xc1, 0xa9, 0x55, 0x99, 0x08, 0xf3, - 0x7c, 0xb9, 0xa5, 0x92, 0x8b, 0x50, 0xf0, 0xbb, 0x3d, 0x31, 0x73, 0x0a, 0x32, 0xea, 0x89, 0x1b, - 0x5b, 0xc8, 0xdb, 0xc8, 0x1b, 0x50, 0x6e, 0x2a, 0x0d, 0xa0, 0x5c, 0xd9, 0xa3, 0xea, 0x0d, 0xb1, - 0xa5, 0xe9, 0x27, 0x0c, 0xb9, 0x71, 0xbb, 0xbd, 0x13, 0xb4, 0x1a, 0xf6, 0x3b, 0x54, 0x79, 0x77, - 0xc2, 0x6e, 0x5f, 0x97, 0x4d, 0xa8, 0x61, 0x7c, 0x8f, 0x6c, 0xd3, 0x7d, 0xe9, 0xdb, 0x14, 0xa3, - 0x3d, 0xf2, 0xa6, 0x6a, 0xc3, 0x10, 0x4a, 0x6a, 0x7a, 0xb1, 0xf0, 0x59, 0x50, 0x94, 0x7e, 0xf9, - 0x1d, 0xde, 0xa0, 0xd6, 0x8d, 0xf1, 0xa5, 0x3c, 0x5c, 0x58, 0xa5, 0x4c, 0x9a, 0x08, 0xcb, 0xb4, - 0xeb, 0x78, 0xfb, 0xdc, 0x4e, 0x43, 0xfa, 0x29, 0xf2, 0x2a, 0x80, 0x1d, 0x6c, 0x37, 0xf6, 0x2c, - 0x31, 0x0d, 0xe5, 0x12, 0xba, 0xa2, 0x56, 0x04, 0xdc, 0x68, 0xd4, 0x15, 0xe4, 0x41, 0xe2, 0x09, - 0x63, 0x34, 0x91, 0xaf, 0x92, 0x7f, 0x88, 0xaf, 0xd2, 0x00, 0xe8, 0x46, 0xd6, 0x5e, 0x41, 0x60, - 0xfe, 0xa4, 0x16, 0x73, 0x14, 0x43, 0x2f, 0xc6, 0x66, 0x0c, 0xfb, 0xcb, 0xf8, 0x93, 0x02, 0xcc, - 0xad, 0x52, 0x16, 0x86, 0x13, 0x94, 0xb2, 0x68, 0x74, 0xa9, 0xc5, 0x47, 0xe5, 0xdd, 0x1c, 0x4c, - 0x38, 0xe6, 0x36, 0x75, 0xb8, 0x32, 0xe7, 0xdc, 0xdf, 0x1a, 0x59, 0x2f, 0x0e, 0x97, 0x32, 0xbf, - 0x26, 0x24, 0xa4, 0x34, 0xa5, 0x6c, 0x44, 0x25, 0x9e, 0xeb, 0x38, 0xcb, 0xe9, 0x05, 0x8c, 0xfa, - 0x1b, 0x9e, 0xcf, 0x94, 0xb1, 0x14, 0xea, 0xb8, 0xa5, 0x08, 0x84, 0x71, 0x3c, 0xb2, 0x00, 0x60, - 0x39, 0x36, 0x75, 0x99, 0xa0, 0x92, 0xd3, 0x8c, 0xe8, 0xf1, 0x5e, 0x0a, 0x21, 0x18, 0xc3, 0xe2, - 0xa2, 0x3a, 0x9e, 0x6b, 0x33, 0x4f, 0x8a, 0x2a, 0x26, 0x45, 0xad, 0x47, 0x20, 0x8c, 0xe3, 0x09, - 0x32, 0xca, 0x7c, 0xdb, 0x0a, 0x04, 0x59, 0x29, 0x45, 0x16, 0x81, 0x30, 0x8e, 0xc7, 0xb7, 0x80, - 0xd8, 0xfb, 0x1f, 0x69, 0x0b, 0xf8, 0xd3, 0x32, 0x5c, 0x4e, 0x0c, 0x2b, 0x33, 0x19, 0xdd, 0xe9, - 0x39, 0x0d, 0xca, 0xf4, 0x07, 0x1c, 0x71, 0x6b, 0xf8, 0xe5, 0xe8, 0xbb, 0xcb, 0x6c, 0xb4, 0x75, - 0x3c, 0xdf, 0x7d, 0xa0, 0x83, 0x87, 0xfa, 0xf6, 0xd7, 0xa0, 0xe2, 0x9a, 0x2c, 0x10, 0x0b, 0x49, - 0xad, 0x99, 0xd0, 0xb1, 0xba, 0xa5, 0x01, 0x18, 0xe1, 0x90, 0x0d, 0x38, 0xa7, 0x86, 0xf8, 0xfa, - 0xfd, 0xae, 0xe7, 0x33, 0xea, 0x4b, 0x5a, 0xb5, 0xbb, 0x28, 0xda, 0x73, 0xeb, 0x19, 0x38, 0x98, - 0x49, 0x49, 0xd6, 0xe1, 0xac, 0x25, 0x33, 0x74, 0xd4, 0xf1, 0xcc, 0xa6, 0x66, 0x28, 0xa3, 0x37, - 0xa1, 0xdd, 0xbf, 0x34, 0x88, 0x82, 0x59, 0x74, 0xe9, 0xd9, 0x3c, 0x31, 0xd2, 0x6c, 0x9e, 0x1c, - 0x65, 0x36, 0x97, 0x47, 0x9b, 0xcd, 0x95, 0xc3, 0xcd, 0x66, 0x3e, 0xf2, 0x7c, 0x1e, 0x51, 0x9f, - 0xef, 0xd6, 0x72, 0xc3, 0x89, 0x25, 0x80, 0xc3, 0x91, 0x6f, 0x64, 0xe0, 0x60, 0x26, 0x25, 0xd9, - 0x86, 0x39, 0xd9, 0x7e, 0xdd, 0xb5, 0xfc, 0xfd, 0x2e, 0xdf, 0x39, 0x62, 0x7c, 0xab, 0x89, 0xf0, - 0xd9, 0x5c, 0x63, 0x28, 0x26, 0x3e, 0x84, 0x0b, 0xf9, 0x19, 0x98, 0x96, 0x5f, 0x69, 0xdd, 0xec, - 0x0a, 0xb6, 0x32, 0x1d, 0x7c, 0x5e, 0xb1, 0x9d, 0x5e, 0x8a, 0x03, 0x31, 0x89, 0x4b, 0x16, 0xe1, - 0x54, 0x77, 0xcf, 0xe2, 0x3f, 0x6f, 0xec, 0xdc, 0xa2, 0xb4, 0x49, 0x9b, 0x22, 0x15, 0x51, 0xa9, - 0x3f, 0xad, 0xbd, 0xf8, 0x8d, 0x24, 0x18, 0xd3, 0xf8, 0xe4, 0x65, 0x98, 0x0a, 0x98, 0xe9, 0x33, - 0x15, 0xb3, 0x9a, 0x9d, 0x91, 0xe9, 0x72, 0x1d, 0xd2, 0x69, 0xc4, 0x60, 0x98, 0xc0, 0x1c, 0x47, - 0x7b, 0x3c, 0x90, 0x9b, 0xa1, 0x08, 0x5c, 0xa7, 0xd4, 0xfe, 0xe7, 0xd3, 0x6a, 0xff, 0xcd, 0x71, - 0x96, 0x7f, 0x86, 0x84, 0x43, 0x2d, 0xfb, 0xd7, 0x81, 0xf8, 0x2a, 0xcc, 0x2e, 0x9d, 0xbb, 0x98, - 0xe6, 0x0f, 0x8b, 0x12, 0x70, 0x00, 0x03, 0x33, 0xa8, 0x48, 0x03, 0xce, 0x07, 0xd4, 0x65, 0xb6, - 0x4b, 0x9d, 0x24, 0x3b, 0xb9, 0x25, 0x3c, 0xab, 0xd8, 0x9d, 0x6f, 0x64, 0x21, 0x61, 0x36, 0xed, - 0x38, 0x83, 0xff, 0x8f, 0x15, 0xb1, 0xef, 0xca, 0xa1, 0x39, 0x36, 0xb5, 0xfd, 0x6e, 0x5a, 0x6d, - 0xbf, 0x35, 0xfe, 0x77, 0x1b, 0x4d, 0x65, 0x2f, 0x00, 0x88, 0xaf, 0x10, 0xd7, 0xd9, 0xa1, 0xa6, - 0xc2, 0x10, 0x82, 0x31, 0x2c, 0xbe, 0x0a, 0xf5, 0x38, 0xc7, 0xd5, 0x75, 0xb8, 0x0a, 0x1b, 0x71, - 0x20, 0x26, 0x71, 0x87, 0xaa, 0xfc, 0xd2, 0xc8, 0x2a, 0xff, 0x75, 0x20, 0x89, 0xd0, 0x82, 0xe4, - 0x37, 0x91, 0xac, 0x89, 0xb9, 0x31, 0x80, 0x81, 0x19, 0x54, 0x43, 0xa6, 0xf2, 0xe4, 0xf1, 0x4e, - 0xe5, 0xf2, 0xe8, 0x53, 0x99, 0xbc, 0x05, 0x17, 0x85, 0x28, 0x35, 0x3e, 0x49, 0xc6, 0x52, 0xf9, - 0xbf, 0x47, 0x31, 0xbe, 0x88, 0xc3, 0x10, 0x71, 0x38, 0x0f, 0xfe, 0x7d, 0x2c, 0x9f, 0x36, 0xb9, - 0x70, 0xd3, 0x19, 0xbe, 0x31, 0x2c, 0x65, 0xe0, 0x60, 0x26, 0x25, 0x9f, 0x62, 0x8c, 0x4f, 0x43, - 0x73, 0xdb, 0xa1, 0x4d, 0x55, 0x13, 0x14, 0x4e, 0xb1, 0xcd, 0xb5, 0x86, 0x82, 0x60, 0x0c, 0x2b, - 0x4b, 0x57, 0x4f, 0x1d, 0x51, 0x57, 0xaf, 0x8a, 0x38, 0xdc, 0x4e, 0x62, 0x4b, 0x50, 0x0a, 0x3f, - 0xac, 0xf2, 0x5a, 0x4a, 0x23, 0xe0, 0x20, 0x8d, 0xd8, 0x2a, 0x2d, 0xdf, 0xee, 0xb2, 0x20, 0xc9, - 0x6b, 0x26, 0xb5, 0x55, 0x66, 0xe0, 0x60, 0x26, 0x25, 0x37, 0x52, 0x76, 0xa9, 0xe9, 0xb0, 0xdd, - 0x24, 0xc3, 0x53, 0x49, 0x23, 0xe5, 0xb5, 0x41, 0x14, 0xcc, 0xa2, 0x1b, 0x47, 0xbd, 0xfd, 0x7a, - 0x1e, 0x2e, 0xae, 0x52, 0x16, 0x66, 0xb2, 0x7f, 0xec, 0x6b, 0xb9, 0x7b, 0xc6, 0xb7, 0xf3, 0x70, - 0x76, 0x95, 0xaa, 0x52, 0xac, 0x0d, 0xaf, 0xa9, 0x95, 0xfd, 0xff, 0xce, 0xe1, 0xe0, 0xb3, 0x35, - 0x2a, 0x66, 0x68, 0x30, 0xcf, 0x97, 0x7b, 0x5d, 0xca, 0xa4, 0x6e, 0x0c, 0xa2, 0x60, 0x16, 0x9d, - 0xf1, 0xef, 0x79, 0x98, 0x5c, 0xf5, 0xbd, 0x5e, 0xb7, 0xbe, 0x4f, 0x5a, 0x30, 0x71, 0x4f, 0xc4, - 0x3c, 0x55, 0x08, 0x72, 0xf4, 0x22, 0x36, 0x19, 0x3a, 0x8d, 0xb6, 0x39, 0xf9, 0x8c, 0x8a, 0x3d, - 0x1f, 0xf8, 0x36, 0xdd, 0xa7, 0xb2, 0x84, 0xa1, 0x1c, 0x0d, 0xfc, 0x4d, 0xde, 0x88, 0x12, 0x46, - 0x3a, 0x70, 0xca, 0x74, 0x1c, 0xef, 0x1e, 0x6d, 0xae, 0x99, 0x8c, 0xba, 0x34, 0xd0, 0x81, 0xe4, - 0xa3, 0x06, 0x52, 0x44, 0x36, 0x66, 0x31, 0xc9, 0x0a, 0xd3, 0xbc, 0xc9, 0xdb, 0x30, 0x19, 0x30, - 0xcf, 0xd7, 0x1b, 0x68, 0x75, 0x61, 0x69, 0xe4, 0xb7, 0xdf, 0xa8, 0x7f, 0xb4, 0x21, 0x59, 0xc9, - 0xd8, 0x8c, 0x7a, 0x40, 0x2d, 0xc0, 0xf8, 0x4a, 0x0e, 0xe0, 0xb5, 0xcd, 0xcd, 0x0d, 0x15, 0x46, - 0x6a, 0x42, 0xd1, 0xec, 0x85, 0xf1, 0xe5, 0xd1, 0x03, 0xbf, 0x89, 0x2a, 0x16, 0x15, 0xab, 0xed, - 0xb1, 0x5d, 0x14, 0xdc, 0xc9, 0xfb, 0x60, 0x52, 0x19, 0x3d, 0x6a, 0xd8, 0xc3, 0x84, 0x90, 0x32, - 0x8c, 0x50, 0xc3, 0x8d, 0xef, 0xe5, 0xe1, 0xc2, 0x0d, 0x97, 0x51, 0xbf, 0xc1, 0x68, 0x37, 0x51, - 0x10, 0x42, 0x7e, 0x7e, 0xa0, 0xc6, 0xfb, 0x27, 0x0e, 0xf7, 0x39, 0x64, 0x89, 0xf0, 0x3a, 0x65, - 0x66, 0xb4, 0xdd, 0x44, 0x6d, 0xb1, 0xc2, 0xee, 0x1e, 0x14, 0x83, 0x2e, 0xb5, 0x54, 0xd4, 0xac, - 0x31, 0xf2, 0x68, 0x64, 0xbf, 0x00, 0xd7, 0x1e, 0x51, 0xa0, 0x5b, 0xe8, 0x12, 0x21, 0x8e, 0x7c, - 0x06, 0x26, 0x02, 0x66, 0xb2, 0x9e, 0x9e, 0x65, 0x5b, 0xc7, 0x2d, 0x58, 0x30, 0x8f, 0x96, 0x84, - 0x7c, 0x46, 0x25, 0xd4, 0xf8, 0x5e, 0x0e, 0xe6, 0xb2, 0x09, 0xd7, 0xec, 0x80, 0x91, 0x9f, 0x1b, - 0x18, 0xf6, 0x43, 0xae, 0x02, 0x4e, 0x2d, 0x06, 0x3d, 0xac, 0x08, 0xd3, 0x2d, 0xb1, 0x21, 0x67, - 0x50, 0xb2, 0x19, 0xed, 0x68, 0xf3, 0xf7, 0xf6, 0x31, 0xbf, 0x7a, 0x4c, 0xb3, 0x72, 0x29, 0x28, - 0x85, 0x19, 0x5f, 0xc8, 0x0f, 0x7b, 0x65, 0xfe, 0x59, 0x88, 0x93, 0x2c, 0x3a, 0xba, 0x39, 0x5e, - 0xd1, 0x51, 0xb2, 0x43, 0x83, 0xb5, 0x47, 0xbf, 0x30, 0x58, 0x7b, 0x74, 0x7b, 0xfc, 0xda, 0xa3, - 0xd4, 0x30, 0x0c, 0x2d, 0x41, 0xfa, 0x95, 0x02, 0x5c, 0x7a, 0xd8, 0xb4, 0xe1, 0xaa, 0x59, 0xcd, - 0xce, 0x71, 0x55, 0xf3, 0xc3, 0xe7, 0x21, 0x59, 0x80, 0x52, 0x77, 0xd7, 0x0c, 0xf4, 0x9e, 0xa8, - 0xed, 0xa9, 0xd2, 0x06, 0x6f, 0x7c, 0xd0, 0xaf, 0x55, 0xe5, 0x5e, 0x2a, 0x1e, 0x51, 0xa2, 0x72, - 0xcd, 0xd2, 0xa1, 0x41, 0x10, 0xb9, 0x2c, 0xa1, 0x66, 0x59, 0x97, 0xcd, 0xa8, 0xe1, 0x84, 0xc1, - 0x84, 0x0c, 0x03, 0x28, 0x25, 0x3b, 0x7a, 0x26, 0x39, 0xa3, 0x4e, 0x2d, 0x7a, 0x29, 0x15, 0x51, - 0x52, 0xb2, 0xc8, 0x3c, 0x14, 0x59, 0x54, 0x35, 0xa4, 0x3d, 0x87, 0x62, 0x86, 0x79, 0x20, 0xf0, - 0x8c, 0xbf, 0x2b, 0xc3, 0x85, 0xec, 0x6f, 0xc8, 0xdf, 0x75, 0x8f, 0xfa, 0x81, 0xed, 0xb9, 0xca, - 0xe4, 0x88, 0x6a, 0x5c, 0x65, 0x33, 0x6a, 0xf8, 0x8f, 0x74, 0x96, 0xfa, 0x0f, 0x72, 0xdc, 0xb3, - 0x91, 0xb1, 0xb7, 0xc7, 0x91, 0xa9, 0x7e, 0x56, 0x7a, 0x48, 0x43, 0x04, 0xe2, 0xf0, 0xbe, 0x90, - 0xdf, 0xcf, 0xc1, 0x6c, 0x27, 0xe5, 0x3a, 0x9d, 0x60, 0x95, 0xb9, 0x28, 0xa5, 0x5b, 0x1f, 0x22, - 0x0f, 0x87, 0xf6, 0x84, 0xfc, 0x22, 0x54, 0xbb, 0x7c, 0x5e, 0x04, 0x8c, 0xba, 0x96, 0x2e, 0x34, - 0x1f, 0x7d, 0xf6, 0x6f, 0x44, 0xbc, 0x74, 0xfe, 0xba, 0x7e, 0xea, 0xa0, 0x5f, 0xab, 0xc6, 0x00, - 0x18, 0x97, 0xf8, 0x84, 0x97, 0x95, 0x5f, 0x85, 0x72, 0x40, 0x19, 0xb3, 0xdd, 0x56, 0x20, 0x1c, - 0xf2, 0x8a, 0x5c, 0x2b, 0x0d, 0xd5, 0x86, 0x21, 0x94, 0xfc, 0x7f, 0xa8, 0x88, 0x50, 0xde, 0xa2, - 0xdf, 0x0a, 0x66, 0x2b, 0x22, 0x2b, 0x2d, 0xf4, 0x6a, 0x43, 0x37, 0x62, 0x04, 0x27, 0x2f, 0xc2, - 0xd4, 0xb6, 0x58, 0xbe, 0xea, 0x78, 0x89, 0x74, 0x9b, 0x45, 0x7e, 0xb1, 0x1e, 0x6b, 0xc7, 0x04, - 0x16, 0x77, 0x91, 0x69, 0x18, 0xef, 0x4c, 0xbb, 0xc8, 0x51, 0x24, 0x14, 0x63, 0x58, 0xe4, 0x59, - 0x28, 0x30, 0x27, 0x10, 0x6e, 0x71, 0x39, 0xb2, 0xda, 0x37, 0xd7, 0x1a, 0xc8, 0xdb, 0x8d, 0xff, - 0xca, 0xc1, 0xa9, 0x54, 0x45, 0x2a, 0x27, 0xe9, 0xf9, 0x8e, 0x52, 0x23, 0x21, 0xc9, 0x16, 0xae, - 0x21, 0x6f, 0x27, 0x6f, 0x29, 0xab, 0x30, 0x3f, 0xe6, 0x49, 0xba, 0x5b, 0x26, 0x0b, 0xb8, 0x19, - 0x38, 0x60, 0x10, 0x8a, 0xf0, 0x69, 0xd4, 0x1f, 0xa5, 0xbb, 0x63, 0xe1, 0xd3, 0x08, 0x86, 0x09, - 0xcc, 0x54, 0x0c, 0xa1, 0x78, 0x98, 0x18, 0x82, 0xf1, 0x37, 0x05, 0xa8, 0xbe, 0xee, 0x6d, 0xff, - 0x88, 0x54, 0x18, 0x65, 0x6b, 0xe4, 0xfc, 0x0f, 0x51, 0x23, 0x6f, 0xc1, 0xd3, 0x8c, 0x39, 0x0d, - 0x6a, 0x79, 0x6e, 0x33, 0x58, 0xdc, 0x61, 0xd4, 0x5f, 0xb1, 0x5d, 0x3b, 0xd8, 0xa5, 0x4d, 0x15, - 0x8c, 0x7d, 0xe6, 0xa0, 0x5f, 0x7b, 0x7a, 0x73, 0x73, 0x2d, 0x0b, 0x05, 0x87, 0xd1, 0x8a, 0x15, - 0x62, 0x5a, 0x6d, 0x6f, 0x67, 0x47, 0x54, 0x92, 0xaa, 0xb4, 0x9d, 0x5c, 0x21, 0xb1, 0x76, 0x4c, - 0x60, 0x19, 0x5f, 0xcb, 0x43, 0xe5, 0xa6, 0xb9, 0xd3, 0x36, 0x1b, 0xb6, 0xdb, 0x26, 0xcf, 0xc1, - 0xe4, 0xb6, 0xef, 0xb5, 0xa9, 0x2f, 0xe3, 0xde, 0xaa, 0x92, 0xb4, 0x2e, 0x9b, 0x50, 0xc3, 0xb8, - 0xd7, 0xc7, 0xbc, 0xae, 0x6d, 0xa5, 0xdd, 0xed, 0x4d, 0xde, 0x88, 0x12, 0x46, 0xee, 0xca, 0x75, - 0x54, 0x18, 0xf3, 0x18, 0xd2, 0xe6, 0x5a, 0x43, 0x26, 0xe4, 0xf5, 0x0a, 0x24, 0xcf, 0x27, 0x2c, - 0x8f, 0xca, 0x50, 0x5b, 0xe1, 0x4d, 0x28, 0x06, 0x66, 0xe0, 0xa8, 0xad, 0x63, 0x8c, 0x43, 0x56, - 0x8b, 0x8d, 0x35, 0x75, 0xc8, 0x6a, 0xb1, 0xb1, 0x86, 0x82, 0xa9, 0xf1, 0x83, 0x3c, 0x54, 0xe5, - 0xb8, 0x49, 0xcf, 0xef, 0x38, 0x47, 0xee, 0x15, 0x91, 0x8d, 0x09, 0x7a, 0x1d, 0xea, 0x0b, 0x87, - 0x5e, 0xad, 0xe7, 0x78, 0x74, 0x2d, 0x02, 0x86, 0x19, 0x99, 0xa8, 0x49, 0x0f, 0x7d, 0xf1, 0x04, - 0x87, 0xbe, 0x74, 0xa8, 0xa1, 0x9f, 0x38, 0x89, 0xa1, 0x7f, 0x37, 0x0f, 0x95, 0x35, 0x7b, 0x87, - 0x5a, 0xfb, 0x96, 0x23, 0x6a, 0xe6, 0x9b, 0xd4, 0xa1, 0x8c, 0xae, 0xfa, 0xa6, 0x45, 0x37, 0xa8, - 0x6f, 0x8b, 0xe3, 0xb2, 0x7c, 0x7d, 0x08, 0x0d, 0xa4, 0x6a, 0xe6, 0x97, 0x87, 0xe0, 0xe0, 0x50, - 0x6a, 0x72, 0x03, 0xa6, 0x9a, 0x34, 0xb0, 0x7d, 0xda, 0xdc, 0x88, 0xd9, 0xd1, 0xcf, 0x69, 0xad, - 0xba, 0x1c, 0x83, 0x3d, 0xe8, 0xd7, 0xa6, 0x37, 0xec, 0x2e, 0x75, 0x6c, 0x97, 0x4a, 0x83, 0x3a, - 0x41, 0xca, 0x97, 0x7c, 0xd7, 0xec, 0x05, 0x59, 0x7d, 0x8c, 0x2d, 0xf9, 0x8d, 0x6c, 0x14, 0x1c, - 0x46, 0x6b, 0x94, 0xa0, 0xb0, 0xe6, 0xb5, 0x8c, 0x2f, 0x14, 0x20, 0x3c, 0x57, 0x4d, 0xbe, 0x98, - 0x83, 0xaa, 0xe9, 0xba, 0x1e, 0x53, 0x67, 0x96, 0x65, 0xfe, 0x0a, 0xc7, 0x3e, 0xbe, 0x3d, 0xbf, - 0x18, 0x31, 0x95, 0xa9, 0x8f, 0x30, 0x1d, 0x13, 0x83, 0x60, 0x5c, 0x36, 0xe9, 0xa5, 0xb2, 0x31, - 0xeb, 0xe3, 0xf7, 0xe2, 0x10, 0xb9, 0x97, 0xb9, 0x8f, 0xc0, 0xe9, 0x74, 0x67, 0x8f, 0x12, 0xbc, - 0x1d, 0x27, 0xee, 0xfb, 0xf9, 0x0a, 0x54, 0x6f, 0x99, 0xcc, 0xde, 0xa3, 0xc2, 0x27, 0x3d, 0x19, - 0x27, 0xe3, 0x77, 0x72, 0x70, 0x21, 0x99, 0x17, 0x39, 0x41, 0x4f, 0x43, 0x9c, 0xa3, 0xc0, 0x4c, - 0x69, 0x38, 0xa4, 0x17, 0xc2, 0xe7, 0x18, 0x48, 0xb3, 0x9c, 0xb4, 0xcf, 0xd1, 0x18, 0x26, 0x10, - 0x87, 0xf7, 0xe5, 0x47, 0xc5, 0xe7, 0x78, 0xb2, 0xcf, 0xb9, 0xa6, 0x3c, 0xa2, 0xc9, 0x27, 0xc6, - 0x23, 0x2a, 0x3f, 0x11, 0x16, 0x68, 0x37, 0xe6, 0x11, 0x55, 0xc6, 0x0c, 0x0c, 0xab, 0x52, 0x02, - 0xc9, 0x6d, 0x98, 0x67, 0x25, 0x2a, 0x83, 0xb5, 0xb3, 0x40, 0x2c, 0x28, 0x6d, 0x9b, 0x81, 0x6d, - 0x29, 0x7b, 0xbc, 0x3e, 0x7a, 0x9c, 0x46, 0x1f, 0x80, 0x94, 0x41, 0x37, 0xf1, 0x88, 0x92, 0x77, - 0x74, 0xd0, 0x32, 0x3f, 0xd6, 0x41, 0x4b, 0xb2, 0x04, 0x45, 0x97, 0x2b, 0xdb, 0xc2, 0x91, 0x8f, - 0x56, 0xde, 0xba, 0x49, 0xf7, 0x51, 0x10, 0x73, 0x9b, 0x16, 0xf8, 0xeb, 0x2b, 0xd3, 0xec, 0x11, - 0xde, 0xd9, 0xfb, 0x60, 0x32, 0xe8, 0x89, 0xf0, 0xb5, 0xda, 0xe1, 0xa3, 0x68, 0xba, 0x6c, 0x46, - 0x0d, 0xe7, 0xd6, 0xdb, 0xa7, 0x7a, 0xb4, 0xa7, 0x83, 0x63, 0xa1, 0xf5, 0xf6, 0x51, 0xde, 0x88, - 0x12, 0x76, 0x72, 0xc6, 0x97, 0x76, 0x23, 0x4b, 0x27, 0xe4, 0x46, 0x1a, 0x9f, 0xcd, 0x03, 0x44, - 0x19, 0x0f, 0xf2, 0x95, 0x1c, 0x9c, 0x0f, 0x57, 0x19, 0x93, 0xc7, 0xaa, 0x96, 0x1c, 0xd3, 0xee, - 0x8c, 0xed, 0xd9, 0x65, 0xad, 0x70, 0xa1, 0x76, 0x36, 0xb2, 0xc4, 0x61, 0x76, 0x2f, 0x08, 0x42, - 0x99, 0x76, 0xba, 0x6c, 0x7f, 0xd9, 0xf6, 0xd5, 0xb4, 0xcb, 0x3c, 0x97, 0x74, 0x5d, 0xe1, 0x48, - 0x52, 0x75, 0x84, 0x46, 0xac, 0x1c, 0x0d, 0xc1, 0x90, 0x8f, 0xf1, 0xe5, 0x3c, 0x9c, 0xcd, 0xe8, - 0x1d, 0x79, 0x15, 0x4e, 0xab, 0x94, 0x4f, 0x74, 0xa7, 0x47, 0x2e, 0xba, 0xd3, 0xa3, 0x91, 0x82, - 0xe1, 0x00, 0x36, 0x79, 0x0b, 0xc0, 0xb4, 0x2c, 0x1a, 0x04, 0xeb, 0x5e, 0x53, 0xdb, 0x92, 0xaf, - 0x70, 0x2f, 0x7b, 0x31, 0x6c, 0x7d, 0xd0, 0xaf, 0x7d, 0x20, 0x2b, 0xf3, 0x98, 0x7a, 0xfb, 0x88, - 0x00, 0x63, 0x2c, 0xc9, 0x27, 0x01, 0xe4, 0x61, 0xb7, 0xb0, 0xa0, 0xf8, 0x11, 0xa9, 0x85, 0x79, - 0x7d, 0x10, 0x6b, 0xfe, 0xa3, 0x3d, 0xd3, 0x65, 0x36, 0xdb, 0x97, 0xa7, 0x31, 0xee, 0x84, 0x5c, - 0x30, 0xc6, 0xd1, 0xf8, 0xab, 0x3c, 0x94, 0xb5, 0x8d, 0xfb, 0x18, 0x92, 0x47, 0xad, 0x44, 0xf2, - 0x68, 0xf4, 0x03, 0x98, 0xba, 0xcb, 0x43, 0xd3, 0x45, 0x5e, 0x2a, 0x5d, 0xb4, 0x3a, 0xbe, 0xa8, - 0x87, 0x27, 0x88, 0xbe, 0x9a, 0x87, 0x19, 0x8d, 0xaa, 0x0e, 0xc5, 0xbe, 0x04, 0xd3, 0x3e, 0x35, - 0x9b, 0x75, 0x93, 0x59, 0xbb, 0xe2, 0xf3, 0xe5, 0x44, 0x01, 0xf7, 0x99, 0x83, 0x7e, 0x6d, 0x1a, - 0xe3, 0x00, 0x4c, 0xe2, 0x91, 0x0f, 0xc3, 0x29, 0x19, 0xf0, 0x5a, 0x37, 0xef, 0xcb, 0x93, 0x29, - 0x62, 0xc0, 0x8a, 0x32, 0x55, 0x5a, 0x4f, 0x82, 0x30, 0x8d, 0xcb, 0xa7, 0xb5, 0x6c, 0xda, 0x0a, - 0xcc, 0x96, 0xec, 0x8c, 0x18, 0x85, 0x69, 0x39, 0xad, 0xeb, 0x29, 0x18, 0x0e, 0x60, 0x13, 0x13, - 0xaa, 0xbc, 0x47, 0x9b, 0x76, 0x87, 0x7a, 0x3d, 0x7d, 0x8d, 0xd1, 0x51, 0xf3, 0xba, 0x62, 0x77, - 0xc7, 0x88, 0x0d, 0xc6, 0x79, 0x1a, 0x7f, 0x9f, 0x83, 0xa9, 0x68, 0xbc, 0x4e, 0x3c, 0x85, 0xb6, - 0x93, 0x4c, 0xa1, 0x2d, 0x8e, 0x3d, 0x1d, 0x86, 0x24, 0xcd, 0x7e, 0x75, 0x32, 0x7a, 0x2d, 0x91, - 0x26, 0xdb, 0x86, 0x39, 0x3b, 0x33, 0x73, 0x14, 0xd3, 0x36, 0x61, 0xa1, 0xe7, 0x8d, 0xa1, 0x98, - 0xf8, 0x10, 0x2e, 0xa4, 0x07, 0xe5, 0x3d, 0xea, 0x33, 0xdb, 0xa2, 0xfa, 0xfd, 0x56, 0xc7, 0xb6, - 0x8e, 0x64, 0x3d, 0x47, 0x34, 0xa6, 0x77, 0x94, 0x00, 0x0c, 0x45, 0x91, 0x6d, 0x28, 0xd1, 0x66, - 0x8b, 0xea, 0xc3, 0x45, 0x63, 0x1e, 0xc4, 0x0f, 0xc7, 0x93, 0x3f, 0x05, 0x28, 0x59, 0x93, 0x00, - 0x2a, 0x8e, 0x8e, 0x0a, 0xa8, 0x79, 0x38, 0xba, 0xad, 0x13, 0xc6, 0x17, 0xa2, 0x42, 0xeb, 0xb0, - 0x09, 0x23, 0x39, 0xa4, 0x1d, 0xde, 0x7e, 0x52, 0x3a, 0x26, 0xe5, 0xf1, 0x90, 0xfb, 0x4f, 0x02, - 0xa8, 0xdc, 0x33, 0x19, 0xf5, 0x3b, 0xa6, 0xdf, 0x56, 0x86, 0xff, 0xe8, 0x6f, 0x78, 0x57, 0x73, - 0x8a, 0xde, 0x30, 0x6c, 0xc2, 0x48, 0x0e, 0xf1, 0xa0, 0xc2, 0x94, 0x25, 0xab, 0xcf, 0x4c, 0x8f, - 0x2e, 0x54, 0xdb, 0xc4, 0x81, 0x8c, 0xf4, 0x87, 0x8f, 0x18, 0xc9, 0x20, 0x7b, 0x89, 0x4b, 0x4a, - 0xe4, 0xd5, 0x34, 0xf5, 0x31, 0x6e, 0x48, 0x52, 0xac, 0xa2, 0xed, 0x26, 0xfb, 0xb2, 0x13, 0xe3, - 0x41, 0x21, 0x52, 0xcb, 0x8f, 0x3b, 0x57, 0xfb, 0x62, 0x32, 0x57, 0x7b, 0x39, 0x9d, 0xab, 0x4d, - 0x05, 0x97, 0x8e, 0x9e, 0xad, 0x35, 0xa1, 0xea, 0x98, 0x01, 0xdb, 0xea, 0x36, 0x4d, 0xa6, 0x02, - 0xfd, 0xd5, 0x85, 0xff, 0x77, 0x38, 0xad, 0xc9, 0xf5, 0x70, 0x14, 0xec, 0x59, 0x8b, 0xd8, 0x60, - 0x9c, 0x27, 0x79, 0x01, 0xaa, 0x7b, 0x42, 0x13, 0xc8, 0x93, 0x4a, 0x25, 0xb1, 0x8d, 0x08, 0xcd, - 0x7e, 0x27, 0x6a, 0xc6, 0x38, 0x0e, 0x27, 0x91, 0x16, 0x48, 0x74, 0x71, 0x83, 0x22, 0x69, 0x44, - 0xcd, 0x18, 0xc7, 0x11, 0x49, 0x23, 0xdb, 0x6d, 0x4b, 0x82, 0x49, 0x41, 0x20, 0x93, 0x46, 0xba, - 0x11, 0x23, 0x38, 0xb9, 0x0a, 0xe5, 0x5e, 0x73, 0x47, 0xe2, 0x96, 0x05, 0xae, 0xb0, 0xfb, 0xb6, - 0x96, 0x57, 0xd4, 0xc9, 0x29, 0x0d, 0x35, 0xbe, 0x9b, 0x03, 0x32, 0x58, 0x5d, 0x40, 0x76, 0x61, - 0xc2, 0x15, 0xd1, 0x9c, 0xb1, 0xef, 0x4b, 0x89, 0x05, 0x85, 0xe4, 0xda, 0x56, 0x0d, 0x8a, 0x3f, - 0x71, 0xa1, 0x4c, 0xef, 0x33, 0xea, 0xbb, 0xa6, 0xa3, 0x4c, 0x9e, 0xe3, 0xb9, 0x9b, 0x45, 0x1a, - 0xba, 0x8a, 0x33, 0x86, 0x32, 0x8c, 0xef, 0xe7, 0xa1, 0x1a, 0xc3, 0x7b, 0x94, 0x93, 0x24, 0xea, - 0xb1, 0x65, 0x10, 0x65, 0xcb, 0x77, 0xd4, 0x34, 0x8d, 0xd5, 0x63, 0x2b, 0x10, 0xae, 0x61, 0x1c, - 0x8f, 0x2c, 0x00, 0x74, 0xcc, 0x80, 0x51, 0x5f, 0x6c, 0x61, 0xa9, 0x2a, 0xe8, 0xf5, 0x10, 0x82, - 0x31, 0x2c, 0x72, 0x45, 0xdd, 0xae, 0x53, 0x4c, 0x9e, 0x64, 0x1d, 0x72, 0x75, 0x4e, 0xe9, 0x18, - 0xae, 0xce, 0x21, 0x2d, 0x38, 0xad, 0x7b, 0xad, 0xa1, 0x47, 0x3b, 0xe7, 0x28, 0x9d, 0x80, 0x14, - 0x0b, 0x1c, 0x60, 0x6a, 0x7c, 0x2d, 0x07, 0xd3, 0x09, 0x17, 0x5e, 0x9e, 0x41, 0xd5, 0xb5, 0x31, - 0x89, 0x33, 0xa8, 0xb1, 0x92, 0x96, 0xe7, 0x61, 0x42, 0x0e, 0x90, 0x1a, 0xf8, 0x50, 0x8d, 0xc8, - 0x21, 0x44, 0x05, 0xe5, 0x0a, 0x41, 0x05, 0x09, 0xd3, 0x0a, 0x41, 0x45, 0x11, 0x51, 0xc3, 0xc9, - 0xfb, 0xa1, 0xac, 0x7b, 0xa7, 0x46, 0x3a, 0xba, 0x68, 0x4a, 0xb5, 0x63, 0x88, 0x61, 0x7c, 0x31, - 0x0f, 0x22, 0x02, 0x4f, 0x5e, 0x82, 0x4a, 0x87, 0x5a, 0xbb, 0xa6, 0x6b, 0x07, 0xfa, 0x44, 0x39, - 0x77, 0xdc, 0x2a, 0xeb, 0xba, 0xf1, 0x01, 0x67, 0xb0, 0xd8, 0x58, 0x13, 0x35, 0x18, 0x11, 0x2e, - 0xb1, 0x60, 0xa2, 0x15, 0x04, 0x66, 0xd7, 0x1e, 0xfb, 0x5a, 0x3d, 0x79, 0xe6, 0x57, 0x2e, 0x22, - 0xf9, 0x1b, 0x15, 0x6b, 0x62, 0x41, 0xa9, 0xeb, 0x98, 0xb6, 0xab, 0x2c, 0xf9, 0xfa, 0x58, 0x79, - 0x87, 0x0d, 0xce, 0x49, 0x86, 0x28, 0xc4, 0x4f, 0x94, 0xbc, 0x8d, 0xff, 0xc8, 0x41, 0x25, 0x84, - 0x93, 0x2d, 0x00, 0x3e, 0x27, 0xd5, 0xb9, 0xd5, 0x23, 0xdd, 0x08, 0x25, 0x9c, 0xad, 0xad, 0x90, - 0x18, 0x63, 0x8c, 0x32, 0x0e, 0xf6, 0xe6, 0x8f, 0xfb, 0x60, 0xef, 0x35, 0xa8, 0xec, 0x9a, 0x6e, - 0x33, 0xd8, 0x35, 0xdb, 0x72, 0x69, 0x96, 0x23, 0x4b, 0xe0, 0x35, 0x0d, 0xc0, 0x08, 0xc7, 0xf8, - 0xd7, 0x12, 0xc8, 0xab, 0xd2, 0xf8, 0xe4, 0x69, 0xda, 0x81, 0xcc, 0x19, 0xe7, 0x04, 0x65, 0x38, - 0x79, 0x96, 0x55, 0x3b, 0x86, 0x18, 0xe4, 0x22, 0x14, 0x3a, 0xb6, 0xab, 0x62, 0xda, 0x22, 0xa4, - 0xb1, 0x6e, 0xbb, 0xc8, 0xdb, 0x04, 0xc8, 0xbc, 0xaf, 0x72, 0x20, 0x12, 0x64, 0xde, 0x47, 0xde, - 0xc6, 0x3d, 0x1b, 0xc7, 0xf3, 0xda, 0xdb, 0xa6, 0xd5, 0xd6, 0xa9, 0x92, 0xa2, 0x50, 0xe1, 0xc2, - 0xb3, 0x59, 0x4b, 0x82, 0x30, 0x8d, 0x4b, 0x56, 0xe1, 0x94, 0xe5, 0x79, 0x4e, 0xd3, 0xbb, 0xe7, - 0x6a, 0x72, 0xb9, 0x23, 0x89, 0x58, 0xf1, 0x32, 0xed, 0xfa, 0xd4, 0xe2, 0xdb, 0xd6, 0x52, 0x12, - 0x09, 0xd3, 0x54, 0x64, 0x0b, 0x9e, 0x7e, 0x87, 0xfa, 0x9e, 0x5a, 0x40, 0x0d, 0x87, 0xd2, 0xae, - 0x66, 0x28, 0xf7, 0x2b, 0x91, 0xba, 0xf9, 0x78, 0x36, 0x0a, 0x0e, 0xa3, 0x15, 0x49, 0x60, 0xd3, - 0x6f, 0x51, 0xb6, 0xe1, 0x7b, 0xdc, 0x85, 0xb7, 0xdd, 0x96, 0x66, 0x3b, 0x19, 0xb1, 0xdd, 0xcc, - 0x46, 0xc1, 0x61, 0xb4, 0xe4, 0x0d, 0x98, 0x95, 0x20, 0xb9, 0x8f, 0x2d, 0xee, 0x99, 0xb6, 0x63, - 0x6e, 0xdb, 0x8e, 0xbe, 0x1f, 0x75, 0x5a, 0x86, 0xa0, 0x37, 0x87, 0xe0, 0xe0, 0x50, 0x6a, 0x71, - 0xab, 0xa9, 0x4a, 0x40, 0x6c, 0x50, 0x5f, 0xcc, 0x03, 0x11, 0xcd, 0x54, 0xae, 0x22, 0xa6, 0x60, - 0x38, 0x80, 0x4d, 0x10, 0x2e, 0x88, 0x2b, 0xf6, 0xb6, 0xba, 0xa9, 0x41, 0x17, 0xc5, 0x1c, 0xd3, - 0x32, 0xd3, 0xd0, 0xc8, 0xc4, 0xc0, 0x21, 0x94, 0xfc, 0x7d, 0x05, 0x64, 0xd9, 0xbb, 0xe7, 0xa6, - 0xb9, 0x56, 0xa3, 0xf7, 0x6d, 0x0c, 0xc1, 0xc1, 0xa1, 0xd4, 0xc6, 0x37, 0xf3, 0x50, 0x09, 0x2d, - 0xc5, 0x43, 0x5c, 0xaf, 0xe0, 0x41, 0x25, 0x4c, 0xf5, 0xab, 0xc5, 0x5a, 0x1f, 0x3f, 0x2a, 0x2f, - 0x8d, 0x9b, 0xf0, 0x11, 0x23, 0x19, 0xf1, 0xfb, 0x1b, 0x0b, 0x63, 0xdc, 0xdf, 0xd8, 0x85, 0x49, - 0xe6, 0xdb, 0xad, 0x96, 0xda, 0x71, 0xab, 0x0b, 0x37, 0xc6, 0xb7, 0xb5, 0x37, 0x25, 0x43, 0x99, - 0x03, 0x57, 0x0f, 0xa8, 0xc5, 0x18, 0x6f, 0xc3, 0xe9, 0x34, 0xa6, 0xd8, 0x8e, 0xac, 0x5d, 0xda, - 0xec, 0x39, 0x7a, 0x8c, 0xa3, 0xed, 0x48, 0xb5, 0x63, 0x88, 0xc1, 0xed, 0x3a, 0x66, 0x77, 0xe8, - 0x3b, 0x9e, 0xab, 0x2d, 0x66, 0xb1, 0xb3, 0x6f, 0xaa, 0x36, 0x0c, 0xa1, 0xc6, 0xbf, 0x14, 0xe0, - 0x62, 0x64, 0xef, 0xaf, 0x9b, 0xae, 0xd9, 0x3a, 0xc4, 0x05, 0x9d, 0x3f, 0xae, 0x5c, 0x39, 0xea, - 0x8d, 0x37, 0x85, 0x27, 0xe0, 0xc6, 0x9b, 0xdf, 0x2a, 0x80, 0xb8, 0x06, 0x97, 0xdc, 0x85, 0x82, - 0xe3, 0xb5, 0xd4, 0x57, 0x1c, 0x3d, 0x02, 0xbf, 0xe6, 0xb5, 0xe4, 0x9e, 0xb4, 0xe6, 0xb5, 0x90, - 0x73, 0xe4, 0xf6, 0x45, 0xdb, 0xdc, 0x69, 0x9b, 0x63, 0xaf, 0xef, 0xb0, 0xe2, 0x46, 0xda, 0x17, - 0xe2, 0x11, 0x25, 0x6f, 0xae, 0x48, 0xb6, 0xf5, 0x3d, 0x8e, 0x63, 0x1b, 0x32, 0xe1, 0x8d, 0x90, - 0x52, 0x91, 0x84, 0x8f, 0x18, 0xc9, 0xe0, 0xa6, 0x59, 0xaf, 0x29, 0xae, 0x23, 0x2e, 0x8e, 0x69, - 0x9a, 0x6d, 0x2d, 0x8b, 0x77, 0x12, 0xa6, 0x99, 0xfc, 0x8d, 0x8a, 0xb5, 0xf1, 0x47, 0x39, 0x98, - 0x6e, 0x38, 0x76, 0xd3, 0x76, 0x5b, 0x27, 0x77, 0x1b, 0x0f, 0xb9, 0x0d, 0xa5, 0xc0, 0xb1, 0x9b, - 0x74, 0xc4, 0x8b, 0x3a, 0xc4, 0xc7, 0xe0, 0xbd, 0xa4, 0x28, 0xf9, 0x18, 0xbf, 0x59, 0x02, 0x75, - 0x77, 0x33, 0xe9, 0x41, 0xa5, 0xa5, 0x6f, 0x0d, 0x51, 0x5d, 0x7e, 0x6d, 0x8c, 0xd3, 0xa5, 0x89, - 0xfb, 0x47, 0xe4, 0xd7, 0x09, 0x1b, 0x31, 0x92, 0x44, 0x68, 0x72, 0xce, 0x2d, 0x8f, 0x39, 0xe7, - 0xa4, 0xb8, 0xc1, 0x59, 0x67, 0x42, 0x71, 0x97, 0xb1, 0xae, 0x9a, 0x70, 0xa3, 0x9f, 0x98, 0x89, - 0x0e, 0xc3, 0xc8, 0xf4, 0x12, 0x7f, 0x46, 0xc1, 0x9a, 0x8b, 0x70, 0xcd, 0xf0, 0xda, 0xc9, 0xa5, - 0xb1, 0xf2, 0x57, 0x71, 0x11, 0xfc, 0x19, 0x05, 0x6b, 0xf2, 0x69, 0xa8, 0x32, 0xdf, 0x74, 0x83, - 0x1d, 0xcf, 0xef, 0x50, 0x5f, 0x39, 0x87, 0x2b, 0x63, 0xcc, 0xe7, 0xcd, 0x88, 0x9b, 0x8c, 0xc6, - 0x27, 0x9a, 0x30, 0x2e, 0x8d, 0xb4, 0xa1, 0xdc, 0x6b, 0xca, 0x8e, 0x29, 0xef, 0x71, 0x71, 0x9c, - 0x95, 0x14, 0x4b, 0x54, 0xe9, 0x27, 0x0c, 0x05, 0x18, 0x1d, 0x50, 0x31, 0x24, 0x62, 0x25, 0x6e, - 0xe8, 0x92, 0xe5, 0x3e, 0xd7, 0x0e, 0x37, 0xf1, 0xc3, 0xcb, 0xa5, 0x62, 0x97, 0x28, 0x64, 0x5e, - 0xc5, 0x65, 0xfc, 0x43, 0x1e, 0x0a, 0x9b, 0x6b, 0x0d, 0x79, 0x26, 0x58, 0x5c, 0x7f, 0x47, 0x1b, - 0x6d, 0xbb, 0x7b, 0x87, 0xfa, 0xf6, 0xce, 0xbe, 0xf2, 0x01, 0x62, 0x67, 0x82, 0xd3, 0x18, 0x98, - 0x41, 0x45, 0xde, 0x84, 0x29, 0xcb, 0x5c, 0xa2, 0x3e, 0x1b, 0xc5, 0xc3, 0x11, 0x75, 0x8d, 0x4b, - 0x8b, 0x11, 0x39, 0x26, 0x98, 0x71, 0xbf, 0xcc, 0x8a, 0x58, 0x17, 0x8e, 0xec, 0x97, 0xc5, 0x18, - 0xc7, 0x18, 0x11, 0x84, 0x4a, 0x9b, 0xa3, 0x0a, 0xae, 0xc5, 0xa3, 0x70, 0x15, 0x2b, 0xfc, 0xa6, - 0xa6, 0xc5, 0x88, 0x8d, 0xe1, 0xc2, 0x74, 0xe2, 0xa2, 0x2f, 0xf2, 0x21, 0x28, 0x7b, 0xdd, 0x98, - 0xa2, 0xa9, 0x08, 0xa7, 0xa5, 0x7c, 0x5b, 0xb5, 0x3d, 0xe8, 0xd7, 0xa6, 0xd7, 0xbc, 0x96, 0x6d, - 0xe9, 0x06, 0x0c, 0xd1, 0x89, 0x01, 0x13, 0xa2, 0x18, 0x49, 0x5f, 0xf3, 0x25, 0x94, 0xa4, 0xb8, - 0x02, 0x28, 0x40, 0x05, 0x31, 0x3e, 0x5b, 0x84, 0x28, 0xf2, 0x4a, 0x02, 0x98, 0x68, 0x8a, 0xeb, - 0x80, 0x94, 0x4e, 0x1b, 0x3d, 0x82, 0x9d, 0xbc, 0x78, 0x50, 0xfa, 0xa0, 0xc9, 0x36, 0x54, 0xa2, - 0x48, 0x0b, 0x0a, 0x6f, 0x7b, 0xdb, 0x63, 0xab, 0xb4, 0x58, 0x15, 0xb2, 0x0c, 0x1b, 0xc6, 0x1a, - 0x90, 0x4b, 0x20, 0xbf, 0x9b, 0x83, 0x33, 0x41, 0xda, 0xfe, 0x53, 0xd3, 0x01, 0xc7, 0x37, 0x74, - 0xd3, 0x16, 0xa5, 0xaa, 0x44, 0x1a, 0x06, 0xc6, 0xc1, 0xbe, 0xf0, 0xf1, 0x97, 0x21, 0x51, 0x35, - 0x9d, 0x56, 0xc7, 0xbc, 0x6a, 0x36, 0x39, 0xfe, 0xc9, 0x36, 0x54, 0xa2, 0x8c, 0x5f, 0xca, 0x43, - 0x35, 0xa6, 0xc7, 0xc6, 0xbe, 0x3d, 0xee, 0x7e, 0xea, 0xf6, 0xb8, 0x8d, 0xd1, 0x33, 0x04, 0x51, - 0xaf, 0x4e, 0xfa, 0x02, 0xb9, 0xbf, 0xce, 0x43, 0x61, 0x6b, 0x79, 0x25, 0xe9, 0xb9, 0xe5, 0x1e, - 0x83, 0xe7, 0xb6, 0x0b, 0x93, 0xdb, 0x3d, 0xdb, 0x61, 0xb6, 0x3b, 0xf6, 0x91, 0x00, 0x7d, 0xd9, - 0x9e, 0x2a, 0x37, 0x96, 0x5c, 0x51, 0xb3, 0x27, 0x2d, 0x98, 0x6c, 0xc9, 0x23, 0xc1, 0x6a, 0xce, - 0xbf, 0x3a, 0xba, 0xc5, 0x22, 0xf9, 0x48, 0x41, 0xea, 0x01, 0x35, 0x77, 0xe3, 0x33, 0xa0, 0x0c, - 0x3e, 0x12, 0x9c, 0xcc, 0x68, 0x86, 0xa1, 0xa9, 0xac, 0x11, 0x35, 0x3e, 0x0d, 0xe1, 0x1e, 0xf9, - 0xd8, 0x3f, 0xa7, 0xf1, 0x6f, 0x39, 0x48, 0x9a, 0x05, 0x8f, 0x7f, 0x46, 0xb5, 0xd3, 0x33, 0x6a, - 0xf9, 0x38, 0x16, 0x60, 0xf6, 0xa4, 0x32, 0xfe, 0x22, 0x0f, 0x13, 0xea, 0xef, 0x54, 0x4e, 0xbe, - 0x0c, 0x84, 0x26, 0xca, 0x40, 0x96, 0xc6, 0x54, 0x8e, 0x43, 0x8b, 0x40, 0x3a, 0xa9, 0x22, 0x90, - 0x71, 0x2f, 0xfc, 0x7e, 0x44, 0x09, 0xc8, 0xdf, 0xe6, 0x40, 0xa9, 0xe6, 0x1b, 0x6e, 0xc0, 0x4c, - 0xd7, 0x12, 0x5e, 0x98, 0xda, 0x07, 0xc6, 0xcd, 0x35, 0xaa, 0x7c, 0xbc, 0xdc, 0xfa, 0xc5, 0x6f, - 0xad, 0xf7, 0xc9, 0xfb, 0xa1, 0xbc, 0xeb, 0x05, 0x4c, 0xe8, 0xfa, 0x7c, 0x32, 0xcc, 0xf2, 0x9a, - 0x6a, 0xc7, 0x10, 0x23, 0x9d, 0x4e, 0x28, 0x0d, 0x4f, 0x27, 0x18, 0x7f, 0x98, 0x87, 0xa9, 0xc4, - 0x35, 0xef, 0x23, 0x57, 0xb4, 0xa4, 0x0a, 0x4a, 0xf2, 0xc7, 0x5f, 0x50, 0x92, 0x55, 0x34, 0x53, - 0x18, 0xb3, 0x68, 0xa6, 0x78, 0x94, 0xa2, 0x19, 0xe3, 0x1b, 0x39, 0x00, 0x3d, 0x5a, 0x27, 0x5e, - 0xcf, 0xd2, 0x4c, 0xd6, 0xb3, 0x8c, 0x3d, 0xaf, 0xb2, 0xab, 0x59, 0xfe, 0xac, 0xa4, 0x5f, 0x49, - 0xd4, 0xb2, 0xbc, 0x9b, 0x83, 0x19, 0x33, 0x51, 0x1f, 0x32, 0xb6, 0x79, 0x99, 0x2a, 0x37, 0x09, - 0xff, 0x70, 0x25, 0xd9, 0x8e, 0x29, 0xb1, 0xe4, 0x65, 0x98, 0xea, 0xaa, 0xe4, 0xf9, 0xad, 0x68, - 0xda, 0x87, 0x67, 0xe3, 0x36, 0x62, 0x30, 0x4c, 0x60, 0x3e, 0xa2, 0x1e, 0xa7, 0x70, 0x2c, 0xf5, - 0x38, 0xf1, 0xa2, 0xff, 0xe2, 0x43, 0x8b, 0xfe, 0xf7, 0xa0, 0xb2, 0xe3, 0x7b, 0x1d, 0x51, 0xf2, - 0xa2, 0xae, 0x0a, 0xbf, 0x3e, 0xc6, 0x9e, 0x12, 0xfd, 0x49, 0x46, 0xb4, 0xb5, 0xae, 0x68, 0xfe, - 0x18, 0x89, 0x12, 0xf1, 0x61, 0x4f, 0x4a, 0x9d, 0x38, 0x4e, 0xa9, 0xa1, 0x2e, 0xd9, 0x94, 0xdc, - 0x51, 0x8b, 0x49, 0x96, 0xb9, 0x4c, 0x3e, 0x9e, 0x32, 0x17, 0xe3, 0x9b, 0xa1, 0x02, 0x6b, 0xa4, - 0x8e, 0xcf, 0xe7, 0x86, 0x1c, 0x9f, 0x57, 0xd7, 0xd8, 0xc4, 0x0b, 0x32, 0x9e, 0x87, 0x09, 0x9f, - 0x9a, 0x81, 0xe7, 0xaa, 0x5b, 0xb2, 0x42, 0xf5, 0x8f, 0xa2, 0x15, 0x15, 0x34, 0x5e, 0xb8, 0x91, - 0x7f, 0x44, 0xe1, 0xc6, 0xfb, 0x63, 0x13, 0x44, 0x56, 0xe6, 0x85, 0x6b, 0x3d, 0x63, 0x92, 0x88, - 0xac, 0xae, 0xfa, 0x17, 0xc5, 0x52, 0x3a, 0xab, 0xab, 0xfe, 0xe1, 0x30, 0xc4, 0x20, 0x4d, 0x98, - 0x72, 0xcc, 0x80, 0x89, 0xe4, 0x48, 0x73, 0x91, 0x8d, 0x50, 0x15, 0x12, 0x2e, 0xa3, 0xb5, 0x18, - 0x1f, 0x4c, 0x70, 0x35, 0xfa, 0x05, 0x48, 0xb9, 0x21, 0x3f, 0x8e, 0xbb, 0xff, 0x8f, 0x8a, 0xbb, - 0xff, 0x46, 0x0e, 0xa2, 0x35, 0x75, 0xc4, 0xdc, 0xf0, 0x1b, 0x50, 0xee, 0x98, 0xf7, 0x97, 0xa9, - 0x63, 0xee, 0x8f, 0x73, 0xb9, 0xf2, 0xba, 0xe2, 0x81, 0x21, 0x37, 0xa3, 0x9f, 0x03, 0x75, 0x59, - 0x11, 0xa1, 0x50, 0xda, 0xb1, 0xef, 0xab, 0xfe, 0x8c, 0x63, 0x1b, 0xc7, 0x2e, 0x93, 0x97, 0x21, - 0x54, 0xd1, 0x80, 0x92, 0x3b, 0xe9, 0xc0, 0x64, 0x20, 0x23, 0xdc, 0xea, 0x55, 0x46, 0x0f, 0x3c, - 0x26, 0x22, 0xe5, 0xea, 0xea, 0x21, 0xd9, 0x84, 0x5a, 0x46, 0xfd, 0x13, 0x5f, 0xff, 0xce, 0xe5, - 0xa7, 0xbe, 0xf1, 0x9d, 0xcb, 0x4f, 0x7d, 0xeb, 0x3b, 0x97, 0x9f, 0xfa, 0xec, 0xc1, 0xe5, 0xdc, - 0xd7, 0x0f, 0x2e, 0xe7, 0xbe, 0x71, 0x70, 0x39, 0xf7, 0xad, 0x83, 0xcb, 0xb9, 0x7f, 0x3a, 0xb8, - 0x9c, 0xfb, 0xb5, 0x7f, 0xbe, 0xfc, 0xd4, 0xc7, 0x5f, 0x1a, 0xf1, 0xbf, 0x78, 0xff, 0x3b, 0x00, - 0x00, 0xff, 0xff, 0xfa, 0x33, 0xe8, 0x93, 0xc5, 0x77, 0x00, 0x00, + // 6637 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5d, 0x5f, 0x6c, 0x24, 0xc9, + 0x59, 0xbf, 0xf9, 0x67, 0xcf, 0x7c, 0x63, 0x7b, 0x77, 0x6b, 0xef, 0xf6, 0xbc, 0xbe, 0xbd, 0xf5, + 0xa6, 0xc3, 0x1d, 0x1b, 0x48, 0xbc, 0x9c, 0xb9, 0x70, 0x17, 0x20, 0xb9, 0xf3, 0xd8, 0x6b, 0x9f, + 0x6f, 0xed, 0x5d, 0xe7, 0x1b, 0x7b, 0xf7, 0x92, 0x23, 0x39, 0xda, 0xdd, 0xe5, 0x71, 0xdf, 0xf4, + 0x74, 0x4f, 0xba, 0x6b, 0xbc, 0xeb, 0x0b, 0x11, 0x09, 0x79, 0xb8, 0x8b, 0x00, 0x05, 0xf1, 0x80, + 0x22, 0x50, 0x40, 0x48, 0x48, 0x3c, 0xa0, 0xbc, 0x20, 0xc2, 0x03, 0x08, 0x01, 0x2f, 0x28, 0xf0, + 0x00, 0x79, 0x40, 0x4a, 0x50, 0x90, 0x45, 0xcc, 0x13, 0x42, 0x44, 0x11, 0x91, 0x50, 0xb4, 0x42, + 0x02, 0xd5, 0xbf, 0xfe, 0x37, 0x3d, 0xbb, 0xf6, 0x8c, 0x7d, 0xd9, 0x40, 0xde, 0xa6, 0xab, 0xbe, + 0xfa, 0x7d, 0xd5, 0xd5, 0x55, 0x5f, 0x7d, 0xff, 0xaa, 0x06, 0x56, 0x5a, 0x0e, 0xdb, 0xed, 0x6d, + 0xcf, 0x59, 0x7e, 0xe7, 0x9a, 0xd7, 0xeb, 0x98, 0xdd, 0xc0, 0x7f, 0x53, 0xfc, 0xd8, 0x71, 0xfd, + 0xbb, 0xd7, 0xba, 0xed, 0xd6, 0x35, 0xb3, 0xeb, 0x84, 0x71, 0xc9, 0xde, 0x73, 0xa6, 0xdb, 0xdd, + 0x35, 0x9f, 0xbb, 0xd6, 0xa2, 0x1e, 0x0d, 0x4c, 0x46, 0xed, 0xb9, 0x6e, 0xe0, 0x33, 0x9f, 0xbc, + 0x10, 0x03, 0xcd, 0x69, 0xa0, 0x39, 0xdd, 0x6c, 0xae, 0xdb, 0x6e, 0xcd, 0x71, 0xa0, 0xb8, 0x44, + 0x03, 0xcd, 0x7c, 0x20, 0xd1, 0x83, 0x96, 0xdf, 0xf2, 0xaf, 0x09, 0xbc, 0xed, 0xde, 0x8e, 0x78, + 0x12, 0x0f, 0xe2, 0x97, 0xe4, 0x33, 0x63, 0xb4, 0x5f, 0x0c, 0xe7, 0x1c, 0x9f, 0x77, 0xeb, 0x9a, + 0xe5, 0x07, 0xf4, 0xda, 0x5e, 0x5f, 0x5f, 0x66, 0x9e, 0x8f, 0x69, 0x3a, 0xa6, 0xb5, 0xeb, 0x78, + 0x34, 0xd8, 0xd7, 0xef, 0x72, 0x2d, 0xa0, 0xa1, 0xdf, 0x0b, 0x2c, 0x7a, 0xac, 0x56, 0xe1, 0xb5, + 0x0e, 0x65, 0x66, 0x1e, 0xaf, 0x6b, 0x83, 0x5a, 0x05, 0x3d, 0x8f, 0x39, 0x9d, 0x7e, 0x36, 0x3f, + 0xf3, 0xb0, 0x06, 0xa1, 0xb5, 0x4b, 0x3b, 0x66, 0xb6, 0x9d, 0xf1, 0xad, 0x1a, 0x9c, 0x5f, 0xd8, + 0x0e, 0x59, 0x60, 0x5a, 0x6c, 0xc3, 0xb7, 0x37, 0x69, 0xa7, 0xeb, 0x9a, 0x8c, 0x92, 0x36, 0x54, + 0x79, 0xdf, 0x6c, 0x93, 0x99, 0xd3, 0x85, 0x2b, 0x85, 0xab, 0xf5, 0xf9, 0x85, 0xb9, 0x21, 0xbf, + 0xc5, 0xdc, 0xba, 0x02, 0x6a, 0x4c, 0x1c, 0x1e, 0xcc, 0x56, 0xf5, 0x13, 0x46, 0x0c, 0xc8, 0x97, + 0x0a, 0x30, 0xe1, 0xf9, 0x36, 0x6d, 0x52, 0x97, 0x5a, 0xcc, 0x0f, 0xa6, 0x8b, 0x57, 0x4a, 0x57, + 0xeb, 0xf3, 0x9f, 0x1c, 0x9a, 0x63, 0xce, 0x1b, 0xcd, 0xdd, 0x4c, 0x30, 0xb8, 0xee, 0xb1, 0x60, + 0xbf, 0xf1, 0xf8, 0xd7, 0x0e, 0x66, 0x1f, 0x3b, 0x3c, 0x98, 0x9d, 0x48, 0x56, 0x61, 0xaa, 0x27, + 0x64, 0x0b, 0xea, 0xcc, 0x77, 0xf9, 0x90, 0x39, 0xbe, 0x17, 0x4e, 0x97, 0x44, 0xc7, 0x2e, 0xcf, + 0xc9, 0xd1, 0xe6, 0xec, 0xe7, 0xf8, 0x74, 0x99, 0xdb, 0x7b, 0x6e, 0x6e, 0x33, 0x22, 0x6b, 0x9c, + 0x57, 0xc0, 0xf5, 0xb8, 0x2c, 0xc4, 0x24, 0x0e, 0xa1, 0x70, 0x26, 0xa4, 0x56, 0x2f, 0x70, 0xd8, + 0xfe, 0xa2, 0xef, 0x31, 0x7a, 0x8f, 0x4d, 0x97, 0xc5, 0x28, 0x3f, 0x9b, 0x07, 0xbd, 0xe1, 0xdb, + 0xcd, 0x34, 0x75, 0xe3, 0xfc, 0xe1, 0xc1, 0xec, 0x99, 0x4c, 0x21, 0x66, 0x31, 0x89, 0x07, 0x67, + 0x9d, 0x8e, 0xd9, 0xa2, 0x1b, 0x3d, 0xd7, 0x6d, 0x52, 0x2b, 0xa0, 0x2c, 0x9c, 0xae, 0x88, 0x57, + 0xb8, 0x9a, 0xc7, 0x67, 0xcd, 0xb7, 0x4c, 0xf7, 0xd6, 0xf6, 0x9b, 0xd4, 0x62, 0x48, 0x77, 0x68, + 0x40, 0x3d, 0x8b, 0x36, 0xa6, 0xd5, 0xcb, 0x9c, 0x5d, 0xcd, 0x20, 0x61, 0x1f, 0x36, 0x59, 0x81, + 0x73, 0xdd, 0xc0, 0xf1, 0x45, 0x17, 0x5c, 0x33, 0x0c, 0x6f, 0x9a, 0x1d, 0x3a, 0x3d, 0x76, 0xa5, + 0x70, 0xb5, 0xd6, 0xb8, 0xa8, 0x60, 0xce, 0x6d, 0x64, 0x09, 0xb0, 0xbf, 0x0d, 0xb9, 0x0a, 0x55, + 0x5d, 0x38, 0x3d, 0x7e, 0xa5, 0x70, 0xb5, 0x22, 0xe7, 0x8e, 0x6e, 0x8b, 0x51, 0x2d, 0x59, 0x86, + 0xaa, 0xb9, 0xb3, 0xe3, 0x78, 0x9c, 0xb2, 0x2a, 0x86, 0xf0, 0x52, 0xde, 0xab, 0x2d, 0x28, 0x1a, + 0x89, 0xa3, 0x9f, 0x30, 0x6a, 0x4b, 0x5e, 0x05, 0x12, 0xd2, 0x60, 0xcf, 0xb1, 0xe8, 0x82, 0x65, + 0xf9, 0x3d, 0x8f, 0x89, 0xbe, 0xd7, 0x44, 0xdf, 0x67, 0x54, 0xdf, 0x49, 0xb3, 0x8f, 0x02, 0x73, + 0x5a, 0x91, 0x97, 0xe1, 0xac, 0x5a, 0x76, 0xf1, 0x28, 0x80, 0x40, 0x7a, 0x9c, 0x0f, 0x24, 0x66, + 0xea, 0xb0, 0x8f, 0x9a, 0xd8, 0x70, 0xc9, 0xec, 0x31, 0xbf, 0xc3, 0x21, 0xd3, 0x4c, 0x37, 0xfd, + 0x36, 0xf5, 0xa6, 0xeb, 0x57, 0x0a, 0x57, 0xab, 0x8d, 0x2b, 0x87, 0x07, 0xb3, 0x97, 0x16, 0x1e, + 0x40, 0x87, 0x0f, 0x44, 0x21, 0xb7, 0xa0, 0x66, 0x7b, 0xe1, 0x86, 0xef, 0x3a, 0xd6, 0xfe, 0xf4, + 0x84, 0xe8, 0xe0, 0x73, 0xea, 0x55, 0x6b, 0x4b, 0x37, 0x9b, 0xb2, 0xe2, 0xfe, 0xc1, 0xec, 0xa5, + 0x7e, 0xe9, 0x38, 0x17, 0xd5, 0x63, 0x8c, 0x41, 0xd6, 0x05, 0xe0, 0xa2, 0xef, 0xed, 0x38, 0xad, + 0xe9, 0x49, 0xf1, 0x35, 0xae, 0x0c, 0x98, 0xd0, 0x4b, 0x37, 0x9b, 0x92, 0xae, 0x31, 0xa9, 0xd8, + 0xc9, 0x47, 0x8c, 0x11, 0x66, 0x5e, 0x82, 0x73, 0x7d, 0xab, 0x96, 0x9c, 0x85, 0x52, 0x9b, 0xee, + 0x0b, 0xa1, 0x54, 0x43, 0xfe, 0x93, 0x3c, 0x0e, 0x95, 0x3d, 0xd3, 0xed, 0xd1, 0xe9, 0xa2, 0x28, + 0x93, 0x0f, 0x3f, 0x5b, 0x7c, 0xb1, 0x60, 0x7c, 0xa7, 0x0e, 0x53, 0x5a, 0x16, 0xdc, 0xa6, 0x01, + 0xa3, 0xf7, 0xc8, 0x15, 0x28, 0x7b, 0xfc, 0x7b, 0x88, 0xf6, 0x8d, 0x09, 0xf5, 0xba, 0x65, 0xf1, + 0x1d, 0x44, 0x0d, 0xb1, 0x60, 0x4c, 0xca, 0x72, 0x81, 0x57, 0x9f, 0x7f, 0x69, 0x68, 0x31, 0xd4, + 0x14, 0x30, 0x0d, 0x38, 0x3c, 0x98, 0x1d, 0x93, 0xbf, 0x51, 0x41, 0x93, 0xd7, 0xa1, 0x1c, 0x3a, + 0x5e, 0x7b, 0xba, 0x24, 0x58, 0x7c, 0x78, 0x78, 0x16, 0x8e, 0xd7, 0x6e, 0x54, 0xf9, 0x1b, 0xf0, + 0x5f, 0x28, 0x40, 0xc9, 0x1d, 0x28, 0xf5, 0xec, 0x1d, 0x25, 0x51, 0x7e, 0x7e, 0x68, 0xec, 0xad, + 0xa5, 0xe5, 0xc6, 0xf8, 0xe1, 0xc1, 0x6c, 0x69, 0x6b, 0x69, 0x19, 0x39, 0x22, 0xf9, 0x62, 0x01, + 0xce, 0x59, 0xbe, 0xc7, 0x4c, 0xbe, 0xbf, 0x68, 0xc9, 0x3a, 0x5d, 0x11, 0x7c, 0x5e, 0x1d, 0x9a, + 0xcf, 0x62, 0x16, 0xb1, 0xf1, 0x04, 0x17, 0x14, 0x7d, 0xc5, 0xd8, 0xcf, 0x9b, 0xfc, 0x4e, 0x01, + 0x9e, 0xe0, 0x0b, 0xb8, 0x8f, 0x58, 0x88, 0x9d, 0x93, 0xed, 0xd5, 0xc5, 0xc3, 0x83, 0xd9, 0x27, + 0x56, 0xf3, 0x98, 0x61, 0x7e, 0x1f, 0x78, 0xef, 0xce, 0x9b, 0xfd, 0x7b, 0x91, 0x10, 0x69, 0xf5, + 0xf9, 0xb5, 0x93, 0xdc, 0xdf, 0x1a, 0x4f, 0xa9, 0xa9, 0x9c, 0xb7, 0x9d, 0x63, 0x5e, 0x2f, 0xc8, + 0x75, 0x18, 0xdf, 0xf3, 0xdd, 0x5e, 0x87, 0x86, 0xd3, 0x55, 0xb1, 0x29, 0xcc, 0xe4, 0xad, 0xd5, + 0xdb, 0x82, 0xa4, 0x71, 0x46, 0xc1, 0x8f, 0xcb, 0xe7, 0x10, 0x75, 0x5b, 0xe2, 0xc0, 0x98, 0xeb, + 0x74, 0x1c, 0x16, 0x0a, 0x69, 0x59, 0x9f, 0xbf, 0x3e, 0xf4, 0x6b, 0xc9, 0x25, 0xba, 0x26, 0xc0, + 0xe4, 0xaa, 0x91, 0xbf, 0x51, 0x31, 0x20, 0x16, 0x54, 0x42, 0xcb, 0x74, 0xa5, 0x34, 0xad, 0xcf, + 0x7f, 0x64, 0xf8, 0x65, 0xc3, 0x51, 0x1a, 0x93, 0xea, 0x9d, 0x2a, 0xe2, 0x11, 0x25, 0x36, 0xf9, + 0x04, 0x4c, 0xa5, 0xbe, 0x66, 0x38, 0x5d, 0x17, 0xa3, 0xf3, 0x74, 0xde, 0xe8, 0x44, 0x54, 0x8d, + 0x0b, 0x0a, 0x6c, 0x2a, 0x35, 0x43, 0x42, 0xcc, 0x80, 0x91, 0x1b, 0x50, 0x0d, 0x1d, 0x9b, 0x5a, + 0x66, 0x10, 0x4e, 0x4f, 0x1c, 0x05, 0xf8, 0xac, 0x02, 0xae, 0x36, 0x55, 0x33, 0x8c, 0x00, 0xc8, + 0x1c, 0x40, 0xd7, 0x0c, 0x98, 0x23, 0xb5, 0x93, 0x49, 0xb1, 0x53, 0x4e, 0x1d, 0x1e, 0xcc, 0xc2, + 0x46, 0x54, 0x8a, 0x09, 0x0a, 0x4e, 0xcf, 0xdb, 0xae, 0x7a, 0xdd, 0x1e, 0x0b, 0xa7, 0xa7, 0xae, + 0x94, 0xae, 0xd6, 0x24, 0x7d, 0x33, 0x2a, 0xc5, 0x04, 0x05, 0xf9, 0x4a, 0x01, 0x9e, 0x8a, 0x1f, + 0xfb, 0x17, 0xd9, 0x99, 0x13, 0x5f, 0x64, 0xb3, 0x87, 0x07, 0xb3, 0x4f, 0x35, 0x07, 0xb3, 0xc4, + 0x07, 0xf5, 0xc7, 0xb8, 0x03, 0x93, 0x0b, 0x3d, 0xb6, 0xeb, 0x07, 0xce, 0x5b, 0x42, 0xd3, 0x22, + 0xcb, 0x50, 0x61, 0x62, 0xc7, 0x94, 0x4a, 0xec, 0x33, 0x79, 0x43, 0x2d, 0xb5, 0x97, 0x1b, 0x74, + 0x5f, 0x6f, 0x34, 0x8d, 0x1a, 0x9f, 0x14, 0x72, 0x07, 0x95, 0xcd, 0x8d, 0xdf, 0x2f, 0x40, 0xad, + 0x61, 0x86, 0x8e, 0xc5, 0xe1, 0xc9, 0x22, 0x94, 0x7b, 0x21, 0x0d, 0x8e, 0x07, 0x2a, 0xa4, 0xf4, + 0x56, 0x48, 0x03, 0x14, 0x8d, 0xc9, 0x2d, 0xa8, 0x76, 0xcd, 0x30, 0xbc, 0xeb, 0x07, 0xb6, 0xda, + 0x69, 0x8e, 0x08, 0x24, 0x55, 0x21, 0xd5, 0x14, 0x23, 0x10, 0xa3, 0x0e, 0xb5, 0x86, 0x6b, 0x5a, + 0xed, 0x5d, 0xdf, 0xa5, 0xc6, 0xf7, 0x0a, 0x70, 0xbe, 0xd1, 0xdb, 0xd9, 0xa1, 0x81, 0xda, 0xf9, + 0xe5, 0x9e, 0x4a, 0x28, 0x54, 0x02, 0x6a, 0x3b, 0xa1, 0xea, 0xfb, 0xd2, 0xd0, 0x9f, 0x0e, 0x39, + 0x8a, 0xda, 0xc2, 0xc5, 0x78, 0x89, 0x02, 0x94, 0xe8, 0xa4, 0x07, 0xb5, 0x37, 0x29, 0x0b, 0x59, + 0x40, 0xcd, 0x8e, 0x7a, 0xbb, 0x57, 0x86, 0x66, 0xf5, 0x2a, 0x65, 0x4d, 0x81, 0x94, 0xd4, 0x18, + 0xa2, 0x42, 0x8c, 0x39, 0x19, 0x7f, 0x5d, 0x81, 0x89, 0x45, 0xbf, 0xb3, 0xed, 0x78, 0xd4, 0xbe, + 0x6e, 0xb7, 0x28, 0x79, 0x03, 0xca, 0xd4, 0x6e, 0x51, 0xf5, 0xb6, 0xc3, 0xef, 0xb3, 0x1c, 0x2c, + 0xd6, 0x16, 0xf8, 0x13, 0x0a, 0x60, 0xb2, 0x06, 0x53, 0x3b, 0x81, 0xdf, 0x91, 0xa2, 0x6b, 0x73, + 0xbf, 0xab, 0xb4, 0x90, 0xc6, 0x8f, 0x69, 0x71, 0xb0, 0x9c, 0xaa, 0xbd, 0x7f, 0x30, 0x0b, 0xf1, + 0x13, 0x66, 0xda, 0x92, 0xd7, 0x60, 0x3a, 0x2e, 0x89, 0xd6, 0xf0, 0x22, 0x57, 0xd9, 0x84, 0xaa, + 0x50, 0x69, 0x5c, 0x3a, 0x3c, 0x98, 0x9d, 0x5e, 0x1e, 0x40, 0x83, 0x03, 0x5b, 0x93, 0xb7, 0x0b, + 0x70, 0x36, 0xae, 0x94, 0x72, 0x55, 0x69, 0x08, 0x27, 0x24, 0xb0, 0x85, 0x6e, 0xbb, 0x9c, 0x61, + 0x81, 0x7d, 0x4c, 0xc9, 0x32, 0x4c, 0x30, 0x3f, 0x31, 0x5e, 0x15, 0x31, 0x5e, 0x86, 0x36, 0xc6, + 0x36, 0xfd, 0x81, 0xa3, 0x95, 0x6a, 0x47, 0x10, 0x2e, 0xe8, 0xe7, 0xcc, 0x48, 0x8d, 0x89, 0x91, + 0x9a, 0x39, 0x3c, 0x98, 0xbd, 0xb0, 0x99, 0x4b, 0x81, 0x03, 0x5a, 0x92, 0xcf, 0x15, 0x60, 0x4a, + 0x57, 0xa9, 0x31, 0x1a, 0x3f, 0xc9, 0x31, 0x22, 0x7c, 0x46, 0x6c, 0xa6, 0x18, 0x60, 0x86, 0xa1, + 0xf1, 0xfd, 0x32, 0xd4, 0x22, 0xc9, 0x46, 0xde, 0x0b, 0x15, 0x61, 0x66, 0x29, 0x85, 0x35, 0xda, + 0xb2, 0x84, 0x35, 0x86, 0xb2, 0x8e, 0x3c, 0x03, 0xe3, 0x96, 0xdf, 0xe9, 0x98, 0x9e, 0x2d, 0x4c, + 0xe7, 0x5a, 0xa3, 0xce, 0x77, 0xea, 0x45, 0x59, 0x84, 0xba, 0x8e, 0x5c, 0x82, 0xb2, 0x19, 0xb4, + 0xa4, 0x15, 0x5b, 0x93, 0xf2, 0x68, 0x21, 0x68, 0x85, 0x28, 0x4a, 0xc9, 0x87, 0xa0, 0x44, 0xbd, + 0xbd, 0xe9, 0xf2, 0x60, 0x55, 0xe0, 0xba, 0xb7, 0x77, 0xdb, 0x0c, 0x1a, 0x75, 0xd5, 0x87, 0xd2, + 0x75, 0x6f, 0x0f, 0x79, 0x1b, 0xb2, 0x06, 0xe3, 0xd4, 0xdb, 0xe3, 0xdf, 0x5e, 0x99, 0x97, 0xef, + 0x19, 0xd0, 0x9c, 0x93, 0x28, 0xad, 0x38, 0x52, 0x28, 0x54, 0x31, 0x6a, 0x08, 0xf2, 0x31, 0x98, + 0x90, 0xba, 0xc5, 0x3a, 0xff, 0x26, 0xe1, 0xf4, 0x98, 0x80, 0x9c, 0x1d, 0xac, 0x9c, 0x08, 0xba, + 0xd8, 0x9c, 0x4f, 0x14, 0x86, 0x98, 0x82, 0x22, 0x1f, 0x83, 0x9a, 0xf6, 0xd4, 0xe8, 0x2f, 0x9b, + 0x6b, 0x09, 0xa3, 0x22, 0x42, 0xfa, 0xa9, 0x9e, 0x13, 0xd0, 0x0e, 0xf5, 0x58, 0xd8, 0x38, 0xa7, + 0x6d, 0x23, 0x5d, 0x1b, 0x62, 0x8c, 0x46, 0xb6, 0xfb, 0x4d, 0x7a, 0x69, 0x8f, 0xbe, 0x77, 0x80, + 0x54, 0x1f, 0xc2, 0x9e, 0xff, 0x24, 0x9c, 0x89, 0x6c, 0x6e, 0x65, 0xb6, 0x49, 0x0b, 0xf5, 0x79, + 0xde, 0x7c, 0x35, 0x5d, 0x75, 0xff, 0x60, 0xf6, 0xe9, 0x1c, 0xc3, 0x2d, 0x26, 0xc0, 0x2c, 0x98, + 0xf1, 0x97, 0x25, 0xe8, 0x57, 0xbb, 0xd3, 0x83, 0x56, 0x38, 0xe9, 0x41, 0xcb, 0xbe, 0x90, 0x14, + 0x9f, 0x2f, 0xaa, 0x66, 0xa3, 0xbf, 0x54, 0xde, 0x87, 0x29, 0x9d, 0xf4, 0x87, 0x79, 0x54, 0xd6, + 0x8e, 0xf1, 0x4e, 0x19, 0xa6, 0x96, 0x4c, 0xda, 0xf1, 0xbd, 0x87, 0x1a, 0x21, 0x85, 0x47, 0xc2, + 0x08, 0xb9, 0x0a, 0xd5, 0x80, 0x76, 0x5d, 0xc7, 0x32, 0x43, 0xf1, 0xe9, 0x95, 0xa7, 0x07, 0x55, + 0x19, 0x46, 0xb5, 0x03, 0x8c, 0xcf, 0xd2, 0x23, 0x69, 0x7c, 0x96, 0x7f, 0xf0, 0xc6, 0xa7, 0xf1, + 0xb9, 0x22, 0x08, 0x45, 0x85, 0x5c, 0x81, 0x32, 0xdf, 0x84, 0xb3, 0x2e, 0x0f, 0x31, 0x71, 0x44, + 0x0d, 0x99, 0x81, 0x22, 0xf3, 0xd5, 0xca, 0x03, 0x55, 0x5f, 0xdc, 0xf4, 0xb1, 0xc8, 0x7c, 0xf2, + 0x16, 0x80, 0xe5, 0x7b, 0xb6, 0xa3, 0x1d, 0xa0, 0xa3, 0xbd, 0xd8, 0xb2, 0x1f, 0xdc, 0x35, 0x03, + 0x7b, 0x31, 0x42, 0x94, 0xe6, 0x47, 0xfc, 0x8c, 0x09, 0x6e, 0xe4, 0x25, 0x18, 0xf3, 0xbd, 0xe5, + 0x9e, 0xeb, 0x8a, 0x01, 0xad, 0x35, 0x7e, 0x9c, 0xdb, 0x84, 0xb7, 0x44, 0xc9, 0xfd, 0x83, 0xd9, + 0x8b, 0x52, 0xbf, 0xe5, 0x4f, 0x77, 0x02, 0x87, 0x39, 0x5e, 0xab, 0xc9, 0x02, 0x93, 0xd1, 0xd6, + 0x3e, 0xaa, 0x66, 0x86, 0x09, 0xf5, 0x65, 0xe7, 0x1e, 0xb5, 0xef, 0x38, 0x9e, 0xed, 0xdf, 0x25, + 0x08, 0x63, 0x2e, 0xf5, 0x5a, 0x6c, 0x57, 0x4d, 0xfe, 0xb9, 0xc4, 0x52, 0x8b, 0xdc, 0xe6, 0x71, + 0xf7, 0x3b, 0x94, 0x99, 0x7c, 0xf1, 0x2d, 0xf5, 0x94, 0x63, 0x57, 0xda, 0xa4, 0x02, 0x01, 0x15, + 0x92, 0xb1, 0x0f, 0xe7, 0xfa, 0x5e, 0x8a, 0xd8, 0x50, 0x66, 0x66, 0x4b, 0x4b, 0xcb, 0xe5, 0xa1, + 0x87, 0x6b, 0xd3, 0x6c, 0x25, 0x86, 0x4a, 0xec, 0xd8, 0x9b, 0x26, 0xdf, 0xb1, 0x39, 0xba, 0xf1, + 0xdf, 0x05, 0xa8, 0x2e, 0xf7, 0x3c, 0x4b, 0x58, 0x3a, 0x0f, 0x77, 0x6c, 0xe9, 0xed, 0xbf, 0x98, + 0xbb, 0xfd, 0xf7, 0x60, 0xac, 0x7d, 0x37, 0x52, 0x0f, 0xea, 0xf3, 0xeb, 0xc3, 0x7f, 0x63, 0xd5, + 0xa5, 0xb9, 0x1b, 0x02, 0x4f, 0x3a, 0xdb, 0xa7, 0x54, 0x87, 0xc6, 0x6e, 0xdc, 0x11, 0x4c, 0x15, + 0xb3, 0x99, 0x0f, 0x41, 0x3d, 0x41, 0x76, 0x2c, 0xef, 0xde, 0x9f, 0x96, 0x61, 0x6c, 0xa5, 0xd9, + 0x5c, 0xd8, 0x58, 0x25, 0x1f, 0x84, 0xba, 0xf2, 0xc3, 0xde, 0x8c, 0xc7, 0x20, 0x72, 0xc3, 0x37, + 0xe3, 0x2a, 0x4c, 0xd2, 0x71, 0xe5, 0x2a, 0xa0, 0xa6, 0xdb, 0x51, 0x53, 0x3f, 0x52, 0xae, 0x90, + 0x17, 0xa2, 0xac, 0x23, 0x26, 0x4c, 0x71, 0x7b, 0x8d, 0x0f, 0xa1, 0xb4, 0xc5, 0xd4, 0x22, 0x38, + 0xa2, 0xb5, 0x26, 0x54, 0xbe, 0xad, 0x14, 0x00, 0x66, 0x00, 0xc9, 0x8b, 0x50, 0x35, 0x7b, 0x6c, + 0x57, 0xa8, 0xc3, 0x72, 0xa6, 0x5f, 0x12, 0x6e, 0x6a, 0x55, 0x76, 0xff, 0x60, 0x76, 0xe2, 0x06, + 0x36, 0x3e, 0xa8, 0x9f, 0x31, 0xa2, 0xe6, 0x9d, 0xd3, 0xf6, 0x9f, 0xea, 0x5c, 0xe5, 0xd8, 0x9d, + 0xdb, 0x48, 0x01, 0x60, 0x06, 0x90, 0xbc, 0x0e, 0x13, 0x6d, 0xba, 0xcf, 0xcc, 0x6d, 0xc5, 0x60, + 0xec, 0x38, 0x0c, 0xce, 0x72, 0x85, 0xec, 0x46, 0xa2, 0x39, 0xa6, 0xc0, 0x48, 0x08, 0x8f, 0xb7, + 0x69, 0xb0, 0x4d, 0x03, 0x5f, 0xd9, 0x92, 0x8a, 0xc9, 0xf8, 0x71, 0x98, 0x4c, 0x1f, 0x1e, 0xcc, + 0x3e, 0x7e, 0x23, 0x07, 0x06, 0x73, 0xc1, 0x8d, 0xef, 0x17, 0xe0, 0xcc, 0x8a, 0x0c, 0x84, 0xf9, + 0x81, 0xdc, 0x52, 0xc9, 0x45, 0x28, 0x05, 0xdd, 0x9e, 0x98, 0x39, 0x25, 0xe9, 0xf5, 0xc4, 0x8d, + 0x2d, 0xe4, 0x65, 0xe4, 0x35, 0xa8, 0xda, 0x4a, 0x02, 0x28, 0x53, 0xf6, 0xb8, 0x72, 0x43, 0x6c, + 0x69, 0xfa, 0x09, 0x23, 0x34, 0xae, 0xb7, 0x77, 0xc2, 0x56, 0xd3, 0x79, 0x8b, 0x2a, 0xeb, 0x4e, + 0xe8, 0xed, 0xeb, 0xb2, 0x08, 0x75, 0x1d, 0xdf, 0x23, 0xdb, 0x74, 0x5f, 0xda, 0x36, 0xe5, 0x78, + 0x8f, 0xbc, 0xa1, 0xca, 0x30, 0xaa, 0x25, 0xb3, 0x7a, 0xb1, 0xf0, 0x59, 0x50, 0x96, 0x76, 0xf9, + 0x6d, 0x5e, 0xa0, 0xd6, 0x8d, 0xf1, 0xc5, 0x22, 0x5c, 0x58, 0xa1, 0x4c, 0xaa, 0x08, 0x4b, 0xb4, + 0xeb, 0xfa, 0xfb, 0x5c, 0x4f, 0x43, 0xfa, 0x29, 0xf2, 0x32, 0x80, 0x13, 0x6e, 0x37, 0xf7, 0x2c, + 0x31, 0x0d, 0xe5, 0x12, 0xba, 0xa2, 0x56, 0x04, 0xac, 0x36, 0x1b, 0xaa, 0xe6, 0x7e, 0xea, 0x09, + 0x13, 0x6d, 0x62, 0x5b, 0xa5, 0xf8, 0x00, 0x5b, 0xa5, 0x09, 0xd0, 0x8d, 0xb5, 0xbd, 0x92, 0xa0, + 0xfc, 0x69, 0xcd, 0xe6, 0x38, 0x8a, 0x5e, 0x02, 0x66, 0x04, 0xfd, 0xcb, 0xf8, 0xb3, 0x12, 0xcc, + 0xac, 0x50, 0x16, 0xb9, 0x13, 0x94, 0xb0, 0x68, 0x76, 0xa9, 0xc5, 0x47, 0xe5, 0xed, 0x02, 0x8c, + 0xb9, 0xe6, 0x36, 0x75, 0xb9, 0x30, 0xe7, 0xe8, 0x6f, 0x0c, 0x2d, 0x17, 0x07, 0x73, 0x99, 0x5b, + 0x13, 0x1c, 0x32, 0x92, 0x52, 0x16, 0xa2, 0x62, 0xcf, 0x65, 0x9c, 0xe5, 0xf6, 0x42, 0x46, 0x83, + 0x0d, 0x3f, 0x60, 0x4a, 0x59, 0x8a, 0x64, 0xdc, 0x62, 0x5c, 0x85, 0x49, 0x3a, 0x32, 0x0f, 0x60, + 0xb9, 0x0e, 0xf5, 0x98, 0x68, 0x25, 0xa7, 0x19, 0xd1, 0xe3, 0xbd, 0x18, 0xd5, 0x60, 0x82, 0x8a, + 0xb3, 0xea, 0xf8, 0x9e, 0xc3, 0x7c, 0xc9, 0xaa, 0x9c, 0x66, 0xb5, 0x1e, 0x57, 0x61, 0x92, 0x4e, + 0x34, 0xa3, 0x2c, 0x70, 0xac, 0x50, 0x34, 0xab, 0x64, 0x9a, 0xc5, 0x55, 0x98, 0xa4, 0xe3, 0x5b, + 0x40, 0xe2, 0xfd, 0x8f, 0xb5, 0x05, 0xfc, 0x79, 0x15, 0x2e, 0xa7, 0x86, 0x95, 0x99, 0x8c, 0xee, + 0xf4, 0xdc, 0x26, 0x65, 0xfa, 0x03, 0x0e, 0xb9, 0x35, 0xfc, 0x6a, 0xfc, 0xdd, 0x65, 0x34, 0xda, + 0x3a, 0x99, 0xef, 0xde, 0xd7, 0xc1, 0x23, 0x7d, 0xfb, 0x6b, 0x50, 0xf3, 0x4c, 0x16, 0x8a, 0x85, + 0xa4, 0xd6, 0x4c, 0x64, 0x58, 0xdd, 0xd4, 0x15, 0x18, 0xd3, 0x90, 0x0d, 0x78, 0x5c, 0x0d, 0xf1, + 0xf5, 0x7b, 0x5d, 0x3f, 0x60, 0x34, 0x90, 0x6d, 0xd5, 0xee, 0xa2, 0xda, 0x3e, 0xbe, 0x9e, 0x43, + 0x83, 0xb9, 0x2d, 0xc9, 0x3a, 0x9c, 0xb7, 0x64, 0x84, 0x8e, 0xba, 0xbe, 0x69, 0x6b, 0x40, 0xe9, + 0xbd, 0x89, 0xf4, 0xfe, 0xc5, 0x7e, 0x12, 0xcc, 0x6b, 0x97, 0x9d, 0xcd, 0x63, 0x43, 0xcd, 0xe6, + 0xf1, 0x61, 0x66, 0x73, 0x75, 0xb8, 0xd9, 0x5c, 0x3b, 0xda, 0x6c, 0xe6, 0x23, 0xcf, 0xe7, 0x11, + 0x0d, 0xf8, 0x6e, 0x2d, 0x37, 0x9c, 0x44, 0x00, 0x38, 0x1a, 0xf9, 0x66, 0x0e, 0x0d, 0xe6, 0xb6, + 0x24, 0xdb, 0x30, 0x23, 0xcb, 0xaf, 0x7b, 0x56, 0xb0, 0xdf, 0xe5, 0x3b, 0x47, 0x02, 0xb7, 0x9e, + 0x72, 0x9f, 0xcd, 0x34, 0x07, 0x52, 0xe2, 0x03, 0x50, 0xc8, 0xcf, 0xc1, 0xa4, 0xfc, 0x4a, 0xeb, + 0x66, 0x57, 0xc0, 0xca, 0x70, 0xf0, 0x13, 0x0a, 0x76, 0x72, 0x31, 0x59, 0x89, 0x69, 0x5a, 0xb2, + 0x00, 0x67, 0xba, 0x7b, 0x16, 0xff, 0xb9, 0xba, 0x73, 0x93, 0x52, 0x9b, 0xda, 0x22, 0x14, 0x51, + 0x6b, 0x3c, 0xa9, 0xad, 0xf8, 0x8d, 0x74, 0x35, 0x66, 0xe9, 0xc9, 0x8b, 0x30, 0x11, 0x32, 0x33, + 0x60, 0xca, 0x67, 0x35, 0x3d, 0x25, 0xc3, 0xe5, 0xda, 0xa5, 0xd3, 0x4c, 0xd4, 0x61, 0x8a, 0x72, + 0x14, 0xe9, 0x71, 0x5f, 0x6e, 0x86, 0xc2, 0x71, 0x9d, 0x11, 0xfb, 0x9f, 0xcf, 0x8a, 0xfd, 0xd7, + 0x47, 0x59, 0xfe, 0x39, 0x1c, 0x8e, 0xb4, 0xec, 0x5f, 0x05, 0x12, 0x28, 0x37, 0xbb, 0x34, 0xee, + 0x12, 0x92, 0x3f, 0x4a, 0x4a, 0xc0, 0x3e, 0x0a, 0xcc, 0x69, 0x45, 0x9a, 0xf0, 0x44, 0x48, 0x3d, + 0xe6, 0x78, 0xd4, 0x4d, 0xc3, 0xc9, 0x2d, 0xe1, 0x69, 0x05, 0xf7, 0x44, 0x33, 0x8f, 0x08, 0xf3, + 0xdb, 0x8e, 0x32, 0xf8, 0xff, 0x5c, 0x13, 0xfb, 0xae, 0x1c, 0x9a, 0x13, 0x13, 0xdb, 0x6f, 0x67, + 0xc5, 0xf6, 0x1b, 0xa3, 0x7f, 0xb7, 0xe1, 0x44, 0xf6, 0x3c, 0x80, 0xf8, 0x0a, 0x49, 0x99, 0x1d, + 0x49, 0x2a, 0x8c, 0x6a, 0x30, 0x41, 0xc5, 0x57, 0xa1, 0x1e, 0xe7, 0xa4, 0xb8, 0x8e, 0x56, 0x61, + 0x33, 0x59, 0x89, 0x69, 0xda, 0x81, 0x22, 0xbf, 0x32, 0xb4, 0xc8, 0x7f, 0x15, 0x48, 0xca, 0xb5, + 0x20, 0xf1, 0xc6, 0xd2, 0x39, 0x31, 0xab, 0x7d, 0x14, 0x98, 0xd3, 0x6a, 0xc0, 0x54, 0x1e, 0x3f, + 0xd9, 0xa9, 0x5c, 0x1d, 0x7e, 0x2a, 0x93, 0x37, 0xe0, 0xa2, 0x60, 0xa5, 0xc6, 0x27, 0x0d, 0x2c, + 0x85, 0xff, 0x7b, 0x14, 0xf0, 0x45, 0x1c, 0x44, 0x88, 0x83, 0x31, 0xf8, 0xf7, 0xb1, 0x02, 0x6a, + 0x73, 0xe6, 0xa6, 0x3b, 0x78, 0x63, 0x58, 0xcc, 0xa1, 0xc1, 0xdc, 0x96, 0x7c, 0x8a, 0x31, 0x3e, + 0x0d, 0xcd, 0x6d, 0x97, 0xda, 0x2a, 0x27, 0x28, 0x9a, 0x62, 0x9b, 0x6b, 0x4d, 0x55, 0x83, 0x09, + 0xaa, 0x3c, 0x59, 0x3d, 0x71, 0x4c, 0x59, 0xbd, 0x22, 0xfc, 0x70, 0x3b, 0xa9, 0x2d, 0x41, 0x09, + 0xfc, 0x28, 0xcb, 0x6b, 0x31, 0x4b, 0x80, 0xfd, 0x6d, 0xc4, 0x56, 0x69, 0x05, 0x4e, 0x97, 0x85, + 0x69, 0xac, 0xa9, 0xcc, 0x56, 0x99, 0x43, 0x83, 0xb9, 0x2d, 0xb9, 0x92, 0xb2, 0x4b, 0x4d, 0x97, + 0xed, 0xa6, 0x01, 0xcf, 0xa4, 0x95, 0x94, 0x57, 0xfa, 0x49, 0x30, 0xaf, 0xdd, 0x28, 0xe2, 0xed, + 0x37, 0x8b, 0x70, 0x71, 0x85, 0xb2, 0x28, 0x92, 0xfd, 0x23, 0x5b, 0xcb, 0xdb, 0x33, 0xbe, 0x55, + 0x84, 0xf3, 0x2b, 0x54, 0xa5, 0x62, 0x6d, 0xf8, 0xb6, 0x16, 0xf6, 0xff, 0x3f, 0x87, 0x83, 0xcf, + 0xd6, 0x38, 0x99, 0xa1, 0xc9, 0xfc, 0x40, 0xee, 0x75, 0x19, 0x95, 0xba, 0xd9, 0x4f, 0x82, 0x79, + 0xed, 0x8c, 0xff, 0x2c, 0xc2, 0xf8, 0x4a, 0xe0, 0xf7, 0xba, 0x8d, 0x7d, 0xd2, 0x82, 0xb1, 0xbb, + 0xc2, 0xe7, 0xa9, 0x5c, 0x90, 0xc3, 0x27, 0xb1, 0x49, 0xd7, 0x69, 0xbc, 0xcd, 0xc9, 0x67, 0x54, + 0xf0, 0x7c, 0xe0, 0xdb, 0x74, 0x9f, 0xca, 0x14, 0x86, 0x6a, 0x3c, 0xf0, 0x37, 0x78, 0x21, 0xca, + 0x3a, 0xd2, 0x81, 0x33, 0xa6, 0xeb, 0xfa, 0x77, 0xa9, 0xbd, 0x66, 0x32, 0xea, 0xd1, 0x50, 0x3b, + 0x92, 0x8f, 0xeb, 0x48, 0x11, 0xd1, 0x98, 0x85, 0x34, 0x14, 0x66, 0xb1, 0xc9, 0x9b, 0x30, 0x1e, + 0x32, 0x3f, 0xd0, 0x1b, 0x68, 0x7d, 0x7e, 0x71, 0xe8, 0xb7, 0xdf, 0x68, 0x7c, 0xb4, 0x29, 0xa1, + 0xa4, 0x6f, 0x46, 0x3d, 0xa0, 0x66, 0x60, 0x7c, 0xb9, 0x00, 0xf0, 0xca, 0xe6, 0xe6, 0x86, 0x72, + 0x23, 0xd9, 0x50, 0x36, 0x7b, 0x91, 0x7f, 0x79, 0x78, 0xc7, 0x6f, 0x2a, 0x8b, 0x45, 0xf9, 0x6a, + 0x7b, 0x6c, 0x17, 0x05, 0x3a, 0x79, 0x1f, 0x8c, 0x2b, 0xa5, 0x47, 0x0d, 0x7b, 0x14, 0x10, 0x52, + 0x8a, 0x11, 0xea, 0x7a, 0xe3, 0x8f, 0x8b, 0x00, 0xab, 0xb6, 0x4b, 0x9b, 0x3a, 0xef, 0xb0, 0xc6, + 0x76, 0x03, 0x1a, 0xee, 0xfa, 0xae, 0x3d, 0xa4, 0x13, 0x5c, 0x64, 0x5f, 0x6c, 0x6a, 0x10, 0x8c, + 0xf1, 0x88, 0xcd, 0x95, 0x78, 0xda, 0x5d, 0xf5, 0x18, 0x0d, 0xf6, 0x4c, 0x77, 0x48, 0x67, 0xd9, + 0x59, 0xa9, 0xf0, 0xc7, 0x38, 0x98, 0x42, 0x25, 0x26, 0xd4, 0x1d, 0xcf, 0x92, 0xf1, 0xc5, 0xc6, + 0xfe, 0x90, 0x13, 0xe9, 0x0c, 0xd7, 0x22, 0x57, 0x63, 0x18, 0x4c, 0x62, 0x1a, 0xdf, 0x2d, 0xc2, + 0x05, 0xc1, 0x8f, 0x77, 0x23, 0x95, 0x45, 0x43, 0x7e, 0xb1, 0x2f, 0x31, 0xfe, 0xa7, 0x8e, 0xc6, + 0x5a, 0xe6, 0x55, 0xaf, 0x53, 0x66, 0xc6, 0x7b, 0x74, 0x5c, 0x96, 0xc8, 0x86, 0xef, 0x41, 0x39, + 0xec, 0x52, 0x4b, 0x8d, 0x5e, 0x73, 0xe8, 0x29, 0x94, 0xff, 0x02, 0x5c, 0xe4, 0xc6, 0xd1, 0x01, + 0x21, 0x80, 0x05, 0x3b, 0xf2, 0x19, 0x18, 0x0b, 0x99, 0xc9, 0x7a, 0x7a, 0x69, 0x6e, 0x9d, 0x34, + 0x63, 0x01, 0x1e, 0xcb, 0x11, 0xf9, 0x8c, 0x8a, 0xa9, 0xf1, 0xdd, 0x02, 0xcc, 0xe4, 0x37, 0x5c, + 0x73, 0x42, 0x46, 0x7e, 0xa1, 0x6f, 0xd8, 0x8f, 0xf8, 0xc5, 0x79, 0x6b, 0x31, 0xe8, 0x51, 0x1a, + 0x9d, 0x2e, 0x49, 0x0c, 0x39, 0x83, 0x8a, 0xc3, 0x68, 0x47, 0xdb, 0x0c, 0xb7, 0x4e, 0xf8, 0xd5, + 0x13, 0xdb, 0x11, 0xe7, 0x82, 0x92, 0x99, 0xf1, 0x4e, 0x71, 0xd0, 0x2b, 0xf3, 0xcf, 0x42, 0xdc, + 0x74, 0xa6, 0xd6, 0x8d, 0xd1, 0x32, 0xb5, 0xd2, 0x1d, 0xea, 0x4f, 0xd8, 0xfa, 0xa5, 0xfe, 0x84, + 0xad, 0x5b, 0xa3, 0x27, 0x6c, 0x65, 0x86, 0x61, 0x60, 0xde, 0xd6, 0xaf, 0x95, 0xe0, 0xd2, 0x83, + 0xa6, 0x0d, 0xdf, 0xcf, 0xd4, 0xec, 0x1c, 0x75, 0x3f, 0x7b, 0xf0, 0x3c, 0x24, 0xf3, 0x50, 0xe9, + 0xee, 0x9a, 0xa1, 0x56, 0x24, 0xb4, 0x12, 0x5a, 0xd9, 0xe0, 0x85, 0xf7, 0xb9, 0xd0, 0x10, 0x0a, + 0x88, 0x78, 0x44, 0x49, 0xca, 0xc5, 0x71, 0x87, 0x86, 0x61, 0x6c, 0xe7, 0x45, 0xe2, 0x78, 0x5d, + 0x16, 0xa3, 0xae, 0x27, 0x0c, 0xc6, 0xa4, 0xef, 0x44, 0xed, 0x4c, 0xc3, 0x87, 0xdf, 0x73, 0x92, + 0xfb, 0xe2, 0x97, 0x52, 0x6e, 0x38, 0xc5, 0x8b, 0xcc, 0x41, 0x99, 0xc5, 0xa9, 0x56, 0xda, 0xdc, + 0x2a, 0xe7, 0xe8, 0x54, 0x82, 0xce, 0xf8, 0x87, 0x2a, 0x5c, 0xc8, 0xff, 0x86, 0xfc, 0x5d, 0xf7, + 0x68, 0x10, 0x3a, 0xbe, 0xa7, 0xf4, 0xb4, 0x38, 0x31, 0x58, 0x16, 0xa3, 0xae, 0xff, 0xa1, 0x0e, + 0xed, 0xff, 0x61, 0x81, 0x9b, 0x83, 0xd2, 0x61, 0xf9, 0x6e, 0x84, 0xf7, 0x9f, 0x96, 0x66, 0xe5, + 0x00, 0x86, 0x38, 0xb8, 0x2f, 0xe4, 0x0f, 0x0a, 0x30, 0xdd, 0xc9, 0xd8, 0x9b, 0xa7, 0x98, 0x9a, + 0x2f, 0xf2, 0x0f, 0xd7, 0x07, 0xf0, 0xc3, 0x81, 0x3d, 0x21, 0xbf, 0x0c, 0xf5, 0x2e, 0x9f, 0x17, + 0x21, 0xa3, 0x9e, 0xa5, 0xb3, 0xf3, 0x87, 0x9f, 0xfd, 0x1b, 0x31, 0x96, 0x0e, 0xfa, 0xcb, 0x3d, + 0x3d, 0x51, 0x81, 0x49, 0x8e, 0x8f, 0x78, 0x2e, 0xfe, 0x55, 0xa8, 0x86, 0x94, 0x31, 0xc7, 0x6b, + 0x85, 0xc2, 0x8b, 0x51, 0x93, 0x6b, 0xa5, 0xa9, 0xca, 0x30, 0xaa, 0x25, 0x3f, 0x09, 0x35, 0xe1, + 0xff, 0x5c, 0x08, 0x5a, 0xe1, 0x74, 0x4d, 0x84, 0xf2, 0x85, 0x5c, 0x6d, 0xea, 0x42, 0x8c, 0xeb, + 0xc9, 0xf3, 0x30, 0xb1, 0x2d, 0x96, 0xaf, 0x3a, 0x93, 0x23, 0x7d, 0x0d, 0x42, 0xc3, 0x6a, 0x24, + 0xca, 0x31, 0x45, 0x45, 0xe6, 0x01, 0x68, 0xe4, 0x24, 0xce, 0xfa, 0x15, 0x62, 0xf7, 0x31, 0x26, + 0xa8, 0xc8, 0xd3, 0x50, 0x62, 0x6e, 0x28, 0x7c, 0x09, 0xd5, 0xd8, 0xd4, 0xd9, 0x5c, 0x6b, 0x22, + 0x2f, 0x37, 0xfe, 0xa7, 0x00, 0x67, 0x32, 0x69, 0xbc, 0xbc, 0x49, 0x2f, 0x70, 0x95, 0x18, 0x89, + 0x9a, 0x6c, 0xe1, 0x1a, 0xf2, 0x72, 0xf2, 0x86, 0x52, 0xa5, 0x8b, 0x23, 0x1e, 0x3f, 0xbc, 0x69, + 0xb2, 0x90, 0xeb, 0xce, 0x7d, 0x5a, 0xb4, 0xf0, 0x39, 0xc7, 0xfd, 0x51, 0xb2, 0x3b, 0xe1, 0x73, + 0x8e, 0xeb, 0x30, 0x45, 0x99, 0x71, 0xbc, 0x94, 0x8f, 0xe2, 0x78, 0x31, 0xfe, 0xae, 0x04, 0xf5, + 0x57, 0xfd, 0xed, 0x1f, 0x92, 0xb4, 0xac, 0x7c, 0x89, 0x5c, 0xfc, 0x01, 0x4a, 0xe4, 0x2d, 0x78, + 0x92, 0x31, 0xb7, 0x49, 0x2d, 0xdf, 0xb3, 0xc3, 0x85, 0x1d, 0x46, 0x83, 0x65, 0xc7, 0x73, 0xc2, + 0x5d, 0x6a, 0x2b, 0x0f, 0xf6, 0x53, 0x87, 0x07, 0xb3, 0x4f, 0x6e, 0x6e, 0xae, 0xe5, 0x91, 0xe0, + 0xa0, 0xb6, 0x62, 0x85, 0x98, 0x56, 0xdb, 0xdf, 0xd9, 0x11, 0xe9, 0xb7, 0x2a, 0xd6, 0x29, 0x57, + 0x48, 0xa2, 0x1c, 0x53, 0x54, 0xc6, 0x57, 0x8b, 0x50, 0xbb, 0x61, 0xee, 0xb4, 0xcd, 0xa6, 0xe3, + 0xb5, 0xc9, 0x33, 0x30, 0xbe, 0x1d, 0xf8, 0x6d, 0x1a, 0xc8, 0x60, 0x81, 0x4a, 0xbf, 0x6d, 0xc8, + 0x22, 0xd4, 0x75, 0xdc, 0x54, 0x66, 0x7e, 0xd7, 0xb1, 0xb2, 0x3e, 0x8a, 0x4d, 0x5e, 0x88, 0xb2, + 0x8e, 0xdc, 0x91, 0xeb, 0xa8, 0x34, 0xe2, 0xd9, 0xad, 0xcd, 0xb5, 0xa6, 0xcc, 0x62, 0xd0, 0x2b, + 0x90, 0x3c, 0x9b, 0xd2, 0x3c, 0x6a, 0x03, 0x75, 0x85, 0xd7, 0xa1, 0x1c, 0x9a, 0xa1, 0xab, 0xb6, + 0x8e, 0x11, 0x4e, 0xa6, 0x2d, 0x34, 0xd7, 0xd4, 0xc9, 0xb4, 0x85, 0xe6, 0x1a, 0x0a, 0x50, 0xe3, + 0xfb, 0x45, 0xa8, 0xcb, 0x71, 0x93, 0xe6, 0xe8, 0x49, 0x8e, 0xdc, 0x4b, 0x22, 0x84, 0x15, 0xf6, + 0x3a, 0x34, 0x10, 0x5e, 0x10, 0xb5, 0x9e, 0x93, 0x2e, 0xc9, 0xb8, 0x32, 0x0a, 0x63, 0xc5, 0x45, + 0x7a, 0xe8, 0xcb, 0xa7, 0x38, 0xf4, 0x95, 0x23, 0x0d, 0xfd, 0xd8, 0x69, 0x0c, 0xfd, 0xdb, 0x45, + 0xa8, 0xad, 0x39, 0x3b, 0xd4, 0xda, 0xb7, 0x5c, 0x71, 0xd0, 0xc0, 0xa6, 0x2e, 0x65, 0x74, 0x25, + 0x30, 0x2d, 0xba, 0x41, 0x03, 0x47, 0x9c, 0x31, 0xe6, 0xeb, 0x43, 0x48, 0x20, 0x75, 0xd0, 0x60, + 0x69, 0x00, 0x0d, 0x0e, 0x6c, 0x4d, 0x56, 0x61, 0xc2, 0xa6, 0xa1, 0x13, 0x50, 0x7b, 0x23, 0xa1, + 0x47, 0x3f, 0xa3, 0xa5, 0xea, 0x52, 0xa2, 0xee, 0xfe, 0xc1, 0xec, 0xe4, 0x86, 0xd3, 0xa5, 0xae, + 0xe3, 0x51, 0xa9, 0x50, 0xa7, 0x9a, 0xf2, 0x25, 0xdf, 0x35, 0x7b, 0x61, 0x5e, 0x1f, 0x13, 0x4b, + 0x7e, 0x23, 0x9f, 0x04, 0x07, 0xb5, 0x35, 0x2a, 0x50, 0x5a, 0xf3, 0x5b, 0xc6, 0x3b, 0x25, 0x88, + 0x0e, 0xa3, 0x93, 0x2f, 0x14, 0xa0, 0x6e, 0x7a, 0x9e, 0xcf, 0xd4, 0x41, 0x6f, 0x19, 0xf4, 0xc3, + 0x91, 0xcf, 0xbc, 0xcf, 0x2d, 0xc4, 0xa0, 0x32, 0x5e, 0x14, 0xc5, 0xb0, 0x12, 0x35, 0x98, 0xe4, + 0x4d, 0x7a, 0x99, 0x10, 0xd6, 0xfa, 0xe8, 0xbd, 0x38, 0x42, 0xc0, 0x6a, 0xe6, 0x23, 0x70, 0x36, + 0xdb, 0xd9, 0xe3, 0x78, 0xbc, 0x47, 0x71, 0x96, 0x7f, 0xbe, 0x06, 0xf5, 0x9b, 0x26, 0x73, 0xf6, + 0xa8, 0xb0, 0x49, 0x4f, 0xc7, 0xc8, 0xf8, 0xdd, 0x02, 0x5c, 0x48, 0x07, 0x93, 0x4e, 0xd1, 0xd2, + 0x10, 0x87, 0x4f, 0x30, 0x97, 0x1b, 0x0e, 0xe8, 0x85, 0xb0, 0x39, 0xfa, 0x62, 0x53, 0xa7, 0x6d, + 0x73, 0x34, 0x07, 0x31, 0xc4, 0xc1, 0x7d, 0xf9, 0x61, 0xb1, 0x39, 0x1e, 0xed, 0xc3, 0xc1, 0x19, + 0x8b, 0x68, 0xfc, 0x91, 0xb1, 0x88, 0xaa, 0x8f, 0x84, 0x06, 0xda, 0x4d, 0x58, 0x44, 0xb5, 0x11, + 0xbd, 0xe9, 0x2a, 0xff, 0x42, 0xa2, 0x0d, 0xb2, 0xac, 0x44, 0x3a, 0xb5, 0x36, 0x16, 0x88, 0x05, + 0x95, 0x6d, 0x33, 0x74, 0x2c, 0xa5, 0x8f, 0x37, 0x86, 0xf7, 0xd3, 0xe8, 0x53, 0xa3, 0xd2, 0xe9, + 0x26, 0x1e, 0x51, 0x62, 0xc7, 0xa7, 0x53, 0x8b, 0x23, 0x9d, 0x4e, 0x25, 0x8b, 0x50, 0xf6, 0xb8, + 0xb0, 0x2d, 0x1d, 0xfb, 0x3c, 0xea, 0xcd, 0x1b, 0x74, 0x1f, 0x45, 0x63, 0xae, 0xd3, 0x02, 0x7f, + 0x7d, 0xa5, 0x9a, 0x3d, 0xc4, 0x3a, 0x7b, 0x1f, 0x8c, 0x87, 0x3d, 0xe1, 0xbe, 0x56, 0x3b, 0x7c, + 0x1c, 0x82, 0x90, 0xc5, 0xa8, 0xeb, 0xb9, 0xf6, 0xf6, 0xa9, 0x1e, 0xed, 0x69, 0xe7, 0x58, 0xa4, + 0xbd, 0x7d, 0x94, 0x17, 0xa2, 0xac, 0x3b, 0x3d, 0xe5, 0x4b, 0x9b, 0x91, 0x95, 0x53, 0x32, 0x23, + 0x8d, 0xcf, 0x16, 0x01, 0xe2, 0x30, 0x11, 0xf9, 0x72, 0x01, 0x9e, 0x88, 0x56, 0x19, 0x93, 0x67, + 0xd1, 0x16, 0x5d, 0xd3, 0xe9, 0x8c, 0x6c, 0xd9, 0xe5, 0xad, 0x70, 0x21, 0x76, 0x36, 0xf2, 0xd8, + 0x61, 0x7e, 0x2f, 0x08, 0x42, 0x95, 0x76, 0xba, 0x6c, 0x7f, 0xc9, 0x09, 0xd4, 0xb4, 0xcb, 0x3d, + 0xcc, 0x75, 0x5d, 0xd1, 0xc8, 0xa6, 0xea, 0xdc, 0x91, 0x58, 0x39, 0xba, 0x06, 0x23, 0x1c, 0xe3, + 0x4b, 0x45, 0x38, 0x9f, 0xd3, 0x3b, 0xf2, 0x32, 0x9c, 0x55, 0x71, 0xb2, 0xf8, 0x22, 0x94, 0x42, + 0x7c, 0x11, 0x4a, 0x33, 0x53, 0x87, 0x7d, 0xd4, 0xe4, 0x0d, 0x00, 0xd3, 0xb2, 0x68, 0x18, 0xae, + 0xfb, 0xb6, 0xd6, 0x25, 0x5f, 0xe2, 0x56, 0xf6, 0x42, 0x54, 0x7a, 0xff, 0x60, 0xf6, 0x03, 0x79, + 0xe1, 0xda, 0xcc, 0xdb, 0xc7, 0x0d, 0x30, 0x01, 0x49, 0x3e, 0x09, 0x20, 0x4f, 0x08, 0x46, 0x59, + 0xd8, 0x0f, 0x09, 0x2d, 0xcc, 0xe9, 0xd3, 0x6b, 0x73, 0x1f, 0xed, 0x99, 0x1e, 0x73, 0xd8, 0xbe, + 0x3c, 0xc2, 0x72, 0x3b, 0x42, 0xc1, 0x04, 0xa2, 0xf1, 0x37, 0x45, 0xa8, 0x6a, 0x1d, 0xf7, 0x5d, + 0x08, 0x1e, 0xb5, 0x52, 0xc1, 0xa3, 0xe1, 0x4f, 0xad, 0xea, 0x2e, 0x0f, 0x0c, 0x17, 0xf9, 0x99, + 0x70, 0xd1, 0xca, 0xe8, 0xac, 0x1e, 0x1c, 0x20, 0xfa, 0x4a, 0x11, 0xa6, 0x34, 0xa9, 0x3a, 0x49, + 0xfc, 0x02, 0x4c, 0x06, 0xd4, 0xb4, 0x1b, 0x26, 0xb3, 0x76, 0xc5, 0xe7, 0x2b, 0x88, 0xac, 0xf7, + 0x73, 0x87, 0x07, 0xb3, 0x93, 0x98, 0xac, 0xc0, 0x34, 0x1d, 0xf9, 0x30, 0x9c, 0x91, 0x0e, 0xaf, + 0x75, 0xf3, 0x9e, 0x3c, 0xce, 0x23, 0x06, 0xac, 0x2c, 0xe3, 0xcb, 0x8d, 0x74, 0x15, 0x66, 0x69, + 0xf9, 0xb4, 0x96, 0x45, 0x5b, 0xa1, 0xd9, 0x92, 0x9d, 0x11, 0xa3, 0x30, 0x29, 0xa7, 0x75, 0x23, + 0x53, 0x87, 0x7d, 0xd4, 0xc4, 0x84, 0x3a, 0xef, 0xd1, 0xa6, 0xd3, 0xa1, 0x7e, 0x4f, 0xdf, 0xfd, + 0x34, 0x54, 0x0c, 0x13, 0x63, 0x18, 0x4c, 0x62, 0x1a, 0xff, 0x58, 0x80, 0x89, 0x78, 0xbc, 0x4e, + 0x3d, 0x84, 0xb6, 0x93, 0x0e, 0xa1, 0x2d, 0x8c, 0x3c, 0x1d, 0x06, 0x04, 0xcd, 0x7e, 0x7d, 0x3c, + 0x7e, 0x2d, 0x11, 0x26, 0xdb, 0x86, 0x19, 0x27, 0x37, 0x72, 0x94, 0x90, 0x36, 0x51, 0x76, 0xec, + 0xea, 0x40, 0x4a, 0x7c, 0x00, 0x0a, 0xe9, 0x41, 0x75, 0x8f, 0x06, 0xcc, 0xb1, 0xa8, 0x7e, 0xbf, + 0x95, 0x91, 0xb5, 0x23, 0x99, 0x04, 0x13, 0x8f, 0xe9, 0x6d, 0xc5, 0x00, 0x23, 0x56, 0x64, 0x1b, + 0x2a, 0xd4, 0x6e, 0x51, 0x7d, 0x22, 0x6b, 0xc4, 0xdb, 0x0b, 0xa2, 0xf1, 0xe4, 0x4f, 0x21, 0x4a, + 0x68, 0x12, 0x42, 0xcd, 0xd5, 0x5e, 0x01, 0x35, 0x0f, 0x87, 0xd7, 0x75, 0x22, 0xff, 0x42, 0x9c, + 0x9d, 0x1e, 0x15, 0x61, 0xcc, 0x87, 0xb4, 0xa3, 0x2b, 0x63, 0x2a, 0x27, 0x24, 0x3c, 0x1e, 0x70, + 0x69, 0x4c, 0x08, 0xb5, 0xbb, 0x26, 0xa3, 0x41, 0xc7, 0x0c, 0xda, 0x4a, 0xf1, 0x1f, 0xfe, 0x0d, + 0xef, 0x68, 0xa4, 0xf8, 0x0d, 0xa3, 0x22, 0x8c, 0xf9, 0x10, 0x1f, 0x6a, 0x4c, 0x69, 0xb2, 0xfa, + 0xa0, 0xf9, 0xf0, 0x4c, 0xb5, 0x4e, 0x1c, 0xaa, 0xdc, 0x0b, 0xfd, 0x88, 0x31, 0x0f, 0xb2, 0x97, + 0xba, 0xd9, 0x45, 0xde, 0xe7, 0xd3, 0x18, 0xe1, 0x5a, 0x29, 0x05, 0x15, 0x6f, 0x37, 0xf9, 0x37, + 0xc4, 0x18, 0xf7, 0x4b, 0xb1, 0x58, 0x7e, 0xb7, 0x63, 0xb5, 0xcf, 0xa7, 0x63, 0xb5, 0x97, 0xb3, + 0xb1, 0xda, 0x8c, 0x73, 0xe9, 0xf8, 0xd1, 0x5a, 0x13, 0xea, 0xae, 0x19, 0xb2, 0xad, 0xae, 0x6d, + 0x32, 0xe5, 0xe8, 0xaf, 0xcf, 0xff, 0xc4, 0xd1, 0xa4, 0x26, 0x97, 0xc3, 0xb1, 0xb3, 0x67, 0x2d, + 0x86, 0xc1, 0x24, 0x26, 0x79, 0x0e, 0xea, 0x7b, 0x42, 0x12, 0xc8, 0xe3, 0x5d, 0x15, 0xb1, 0x8d, + 0x08, 0xc9, 0x7e, 0x3b, 0x2e, 0xc6, 0x24, 0x0d, 0x6f, 0x22, 0x35, 0x90, 0xf8, 0xb6, 0x0b, 0xd5, + 0xa4, 0x19, 0x17, 0x63, 0x92, 0x46, 0x04, 0x8d, 0x1c, 0xaf, 0x2d, 0x1b, 0x8c, 0x8b, 0x06, 0x32, + 0x68, 0xa4, 0x0b, 0x31, 0xae, 0x27, 0x57, 0xa1, 0xda, 0xb3, 0x77, 0x24, 0x6d, 0x55, 0xd0, 0x0a, + 0xbd, 0x6f, 0x6b, 0x69, 0x59, 0x1d, 0x37, 0xd3, 0xb5, 0xc6, 0x77, 0x0a, 0x40, 0xfa, 0xb3, 0x0b, + 0xc8, 0x2e, 0x8c, 0x79, 0xc2, 0x9b, 0x33, 0xf2, 0x25, 0x33, 0x09, 0xa7, 0x90, 0x5c, 0xdb, 0xaa, + 0x40, 0xe1, 0x13, 0x0f, 0xaa, 0xf4, 0x1e, 0xa3, 0x81, 0x17, 0x65, 0x1b, 0x9d, 0xcc, 0x85, 0x36, + 0x52, 0xd1, 0x55, 0xc8, 0x18, 0xf1, 0x30, 0xbe, 0x57, 0x84, 0x7a, 0x82, 0xee, 0x61, 0x46, 0x92, + 0x48, 0x62, 0x97, 0x4e, 0x94, 0xad, 0xc0, 0x55, 0xd3, 0x34, 0x91, 0xc4, 0xae, 0xaa, 0x70, 0x0d, + 0x93, 0x74, 0x64, 0x1e, 0xa0, 0x63, 0x86, 0x8c, 0x06, 0x62, 0x0b, 0xcb, 0xa4, 0x8e, 0xaf, 0x47, + 0x35, 0x98, 0xa0, 0x22, 0x57, 0xd4, 0x95, 0x44, 0xe5, 0xf4, 0xf1, 0xdf, 0x01, 0xf7, 0x0d, 0x55, + 0x4e, 0xe0, 0xbe, 0x21, 0xd2, 0x82, 0xb3, 0xba, 0xd7, 0xba, 0xf6, 0x78, 0x87, 0x43, 0xa5, 0x11, + 0x90, 0x81, 0xc0, 0x3e, 0x50, 0xe3, 0xab, 0x05, 0x98, 0x4c, 0x99, 0xf0, 0xf2, 0xe0, 0xae, 0xce, + 0x8d, 0x49, 0x1d, 0xdc, 0x4d, 0xa4, 0xb4, 0x3c, 0x0b, 0x63, 0x72, 0x80, 0xd4, 0xc0, 0x47, 0x62, + 0x44, 0x0e, 0x21, 0xaa, 0x5a, 0x2e, 0x10, 0x94, 0x93, 0x30, 0x2b, 0x10, 0x94, 0x17, 0x11, 0x75, + 0x3d, 0x79, 0x3f, 0x54, 0x75, 0xef, 0xd4, 0x48, 0xc7, 0xb7, 0x73, 0xa9, 0x72, 0x8c, 0x28, 0x8c, + 0x2f, 0x14, 0x41, 0x78, 0xe0, 0xc9, 0x0b, 0x50, 0xeb, 0x50, 0x6b, 0xd7, 0xf4, 0x9c, 0x50, 0x1f, + 0xc3, 0xe7, 0x86, 0x5b, 0x6d, 0x5d, 0x17, 0xde, 0xe7, 0x00, 0x0b, 0xcd, 0x35, 0x91, 0x83, 0x11, + 0xd3, 0x12, 0x0b, 0xc6, 0x5a, 0x61, 0x68, 0x76, 0x9d, 0x91, 0xef, 0x22, 0x94, 0x07, 0xa5, 0xe5, + 0x22, 0x92, 0xbf, 0x51, 0x41, 0x13, 0x0b, 0x2a, 0x5d, 0xd7, 0x74, 0x3c, 0xa5, 0xc9, 0x37, 0x46, + 0x8a, 0x3b, 0x6c, 0x70, 0x24, 0xe9, 0xa2, 0x10, 0x3f, 0x51, 0x62, 0x1b, 0xff, 0x55, 0x80, 0x5a, + 0x54, 0x4f, 0xb6, 0x00, 0xf8, 0x9c, 0x54, 0x87, 0x7d, 0x8f, 0x75, 0x8d, 0x96, 0x30, 0xb6, 0xb6, + 0xa2, 0xc6, 0x98, 0x00, 0xca, 0x39, 0x0d, 0x5d, 0x3c, 0xe9, 0xd3, 0xd0, 0xd7, 0xa0, 0xb6, 0x6b, + 0x7a, 0x76, 0xb8, 0x6b, 0xb6, 0xe5, 0xd2, 0xac, 0xc6, 0x9a, 0xc0, 0x2b, 0xba, 0x02, 0x63, 0x1a, + 0xe3, 0xdf, 0x2b, 0x20, 0xef, 0x97, 0xe3, 0x93, 0xc7, 0x76, 0x42, 0x19, 0x33, 0x2e, 0x88, 0x96, + 0xd1, 0xe4, 0x59, 0x52, 0xe5, 0x18, 0x51, 0x90, 0x8b, 0x50, 0xea, 0x38, 0x9e, 0xf2, 0x69, 0x0b, + 0x97, 0xc6, 0xba, 0xe3, 0x21, 0x2f, 0x13, 0x55, 0xe6, 0x3d, 0x15, 0x03, 0x91, 0x55, 0xe6, 0x3d, + 0xe4, 0x65, 0xdc, 0xb2, 0x71, 0x7d, 0xbf, 0xbd, 0x6d, 0x5a, 0x6d, 0x1d, 0x2a, 0x29, 0x0b, 0x11, + 0x2e, 0x2c, 0x9b, 0xb5, 0x74, 0x15, 0x66, 0x69, 0xc9, 0x0a, 0x9c, 0xb1, 0x7c, 0xdf, 0xb5, 0xfd, + 0xbb, 0x9e, 0x6e, 0x2e, 0x77, 0x24, 0xe1, 0x2b, 0x5e, 0xa2, 0xdd, 0x80, 0x5a, 0x7c, 0xdb, 0x5a, + 0x4c, 0x13, 0x61, 0xb6, 0x15, 0xd9, 0x82, 0x27, 0xdf, 0xa2, 0x81, 0xaf, 0x16, 0x50, 0xd3, 0xa5, + 0xb4, 0xab, 0x01, 0xe5, 0x7e, 0x25, 0x42, 0x37, 0x1f, 0xcf, 0x27, 0xc1, 0x41, 0x6d, 0x45, 0x10, + 0xd8, 0x0c, 0x5a, 0x94, 0x6d, 0x04, 0x3e, 0x37, 0xe1, 0x1d, 0xaf, 0xa5, 0x61, 0xc7, 0x63, 0xd8, + 0xcd, 0x7c, 0x12, 0x1c, 0xd4, 0x96, 0xbc, 0x06, 0xd3, 0xb2, 0x4a, 0xee, 0x63, 0x0b, 0x7b, 0xa6, + 0xe3, 0x9a, 0xdb, 0x8e, 0xab, 0x2f, 0x95, 0x9d, 0x94, 0x2e, 0xe8, 0xcd, 0x01, 0x34, 0x38, 0xb0, + 0xb5, 0xb8, 0x0a, 0x56, 0x05, 0x20, 0x36, 0x68, 0x20, 0xe6, 0x81, 0xf0, 0x66, 0x2a, 0x53, 0x11, + 0x33, 0x75, 0xd8, 0x47, 0x4d, 0x10, 0x2e, 0x88, 0x7b, 0x09, 0xb7, 0xba, 0x99, 0x41, 0x17, 0xc9, + 0x1c, 0x93, 0x32, 0xd2, 0xd0, 0xcc, 0xa5, 0xc0, 0x01, 0x2d, 0xf9, 0xfb, 0x8a, 0x9a, 0x25, 0xff, + 0xae, 0x97, 0x45, 0xad, 0xc7, 0xef, 0xdb, 0x1c, 0x40, 0x83, 0x03, 0x5b, 0x1b, 0xdf, 0x28, 0x42, + 0x2d, 0xd2, 0x14, 0x8f, 0x70, 0x27, 0x85, 0x0f, 0xb5, 0x28, 0xd4, 0xaf, 0x16, 0x6b, 0x63, 0x74, + 0xaf, 0xbc, 0x54, 0x6e, 0xa2, 0x47, 0x8c, 0x79, 0x24, 0x2f, 0xbd, 0x2c, 0x8d, 0x70, 0xe9, 0x65, + 0x17, 0xc6, 0x59, 0xe0, 0xb4, 0x5a, 0x6a, 0xc7, 0xad, 0xcf, 0xaf, 0x8e, 0xae, 0x6b, 0x6f, 0x4a, + 0x40, 0x19, 0x03, 0x57, 0x0f, 0xa8, 0xd9, 0x18, 0x6f, 0xc2, 0xd9, 0x2c, 0xa5, 0xd8, 0x8e, 0xac, + 0x5d, 0x6a, 0xf7, 0x5c, 0x3d, 0xc6, 0xf1, 0x76, 0xa4, 0xca, 0x31, 0xa2, 0xe0, 0x7a, 0x1d, 0x73, + 0x3a, 0xf4, 0x2d, 0xdf, 0xd3, 0x1a, 0xb3, 0xd8, 0xd9, 0x37, 0x55, 0x19, 0x46, 0xb5, 0xc6, 0xbf, + 0x95, 0xe0, 0x62, 0xac, 0xef, 0xaf, 0x9b, 0x9e, 0xd9, 0x3a, 0xc2, 0xad, 0xa6, 0x3f, 0xca, 0x5c, + 0x39, 0xee, 0x35, 0x41, 0xa5, 0x47, 0xe0, 0x9a, 0xa0, 0xdf, 0x2e, 0x81, 0xb8, 0x3b, 0x98, 0xdc, + 0x81, 0x92, 0xeb, 0xb7, 0xd4, 0x57, 0x1c, 0xde, 0x03, 0xbf, 0xe6, 0xb7, 0xe4, 0x9e, 0xb4, 0xe6, + 0xb7, 0x90, 0x23, 0x72, 0xfd, 0xa2, 0x6d, 0xee, 0xb4, 0xcd, 0x91, 0xd7, 0x77, 0x94, 0x71, 0x23, + 0xf5, 0x0b, 0xf1, 0x88, 0x12, 0x9b, 0x0b, 0x92, 0x6d, 0x7d, 0xf9, 0xe5, 0xc8, 0x8a, 0x4c, 0x74, + 0x8d, 0xa6, 0x14, 0x24, 0xd1, 0x23, 0xc6, 0x3c, 0xb8, 0x6a, 0xd6, 0xb3, 0xc5, 0x1d, 0xce, 0xe5, + 0x11, 0x55, 0xb3, 0xad, 0x25, 0xf1, 0x4e, 0x42, 0x35, 0x93, 0xbf, 0x51, 0x41, 0x1b, 0x7f, 0x52, + 0x80, 0xc9, 0xa6, 0xeb, 0xd8, 0x8e, 0xd7, 0x3a, 0xbd, 0x2b, 0x8c, 0xc8, 0x2d, 0xa8, 0x84, 0xae, + 0x63, 0xd3, 0x21, 0x0f, 0x6c, 0x88, 0x8f, 0xc1, 0x7b, 0x49, 0x51, 0xe2, 0x18, 0xbf, 0x55, 0x01, + 0x75, 0xe1, 0x35, 0xe9, 0x41, 0xad, 0xa5, 0xaf, 0x5a, 0x51, 0x5d, 0x7e, 0x65, 0x84, 0x23, 0xb9, + 0xa9, 0x4b, 0x5b, 0xe4, 0xd7, 0x89, 0x0a, 0x31, 0xe6, 0x44, 0x68, 0x7a, 0xce, 0x2d, 0x8d, 0x38, + 0xe7, 0x24, 0xbb, 0xfe, 0x59, 0x67, 0x42, 0x79, 0x97, 0xb1, 0xae, 0x9a, 0x70, 0xc3, 0x1f, 0x33, + 0x8a, 0x4f, 0x10, 0xc9, 0xf0, 0x12, 0x7f, 0x46, 0x01, 0xcd, 0x59, 0x78, 0x66, 0x74, 0x57, 0xe7, + 0xe2, 0x48, 0xf1, 0xab, 0x24, 0x0b, 0xfe, 0x8c, 0x02, 0x9a, 0x7c, 0x1a, 0xea, 0x2c, 0x30, 0xbd, + 0x70, 0xc7, 0x0f, 0x3a, 0x34, 0x50, 0xc6, 0xe1, 0xf2, 0x08, 0xf3, 0x79, 0x33, 0x46, 0x93, 0xde, + 0xf8, 0x54, 0x11, 0x26, 0xb9, 0x91, 0x36, 0x54, 0x7b, 0xb6, 0xec, 0x98, 0xb2, 0x1e, 0x17, 0x46, + 0x59, 0x49, 0x89, 0x40, 0x95, 0x7e, 0xc2, 0x88, 0x81, 0xd1, 0x01, 0xe5, 0x43, 0x22, 0x56, 0xea, + 0x5a, 0x33, 0x99, 0xee, 0x73, 0xed, 0x68, 0x13, 0x3f, 0xba, 0x91, 0x2b, 0x71, 0xf3, 0x44, 0xee, + 0xfd, 0x65, 0xc6, 0x3f, 0x15, 0xa1, 0xb4, 0xb9, 0xd6, 0x94, 0x07, 0xa9, 0xc5, 0x9d, 0x81, 0xb4, + 0xd9, 0x76, 0xba, 0xb7, 0x69, 0xe0, 0xec, 0xec, 0x2b, 0x1b, 0x20, 0x71, 0x90, 0x3a, 0x4b, 0x81, + 0x39, 0xad, 0xc8, 0xeb, 0x30, 0x61, 0x99, 0x8b, 0x34, 0x60, 0xc3, 0x58, 0x38, 0x22, 0xaf, 0x71, + 0x71, 0x21, 0x6e, 0x8e, 0x29, 0x30, 0x6e, 0x97, 0x59, 0x31, 0x74, 0xe9, 0xd8, 0x76, 0x59, 0x02, + 0x38, 0x01, 0x44, 0x10, 0x6a, 0x6d, 0x4e, 0x2a, 0x50, 0xcb, 0xc7, 0x41, 0x15, 0x2b, 0xfc, 0x86, + 0x6e, 0x8b, 0x31, 0x8c, 0xe1, 0xc1, 0x64, 0xea, 0x76, 0x34, 0xf2, 0x21, 0xa8, 0xfa, 0xdd, 0x84, + 0xa0, 0xa9, 0x09, 0xa3, 0xa5, 0x7a, 0x4b, 0x95, 0xdd, 0x3f, 0x98, 0x9d, 0x5c, 0xf3, 0x5b, 0x8e, + 0xa5, 0x0b, 0x30, 0x22, 0x27, 0x06, 0x8c, 0x89, 0x64, 0x24, 0x7d, 0x37, 0x9a, 0x10, 0x92, 0xe2, + 0xde, 0xa4, 0x10, 0x55, 0x8d, 0xf1, 0xd9, 0x32, 0xc4, 0x9e, 0x57, 0x12, 0xc2, 0x98, 0x2d, 0xee, + 0x50, 0x52, 0x32, 0x6d, 0x78, 0x0f, 0x76, 0xfa, 0xb6, 0x46, 0x69, 0x83, 0xa6, 0xcb, 0x50, 0xb1, + 0x22, 0x2d, 0x28, 0xbd, 0xe9, 0x6f, 0x8f, 0x2c, 0xd2, 0x12, 0x59, 0xc8, 0xd2, 0x6d, 0x98, 0x28, + 0x40, 0xce, 0x81, 0xfc, 0x5e, 0x01, 0xce, 0x85, 0x59, 0xfd, 0x4f, 0x4d, 0x07, 0x1c, 0x5d, 0xd1, + 0xcd, 0x6a, 0x94, 0x2a, 0x13, 0x69, 0x50, 0x35, 0xf6, 0xf7, 0x85, 0x8f, 0xbf, 0x74, 0x89, 0xaa, + 0xe9, 0xb4, 0x32, 0xe2, 0xfd, 0xbc, 0xe9, 0xf1, 0x4f, 0x97, 0xa1, 0x62, 0x65, 0xfc, 0x4a, 0x11, + 0xea, 0x09, 0x39, 0x36, 0xf2, 0x95, 0x7b, 0xf7, 0x32, 0x57, 0xee, 0x6d, 0x0c, 0x1f, 0x21, 0x88, + 0x7b, 0x75, 0xda, 0xb7, 0xee, 0xfd, 0x6d, 0x11, 0x4a, 0x5b, 0x4b, 0xcb, 0x69, 0xcb, 0xad, 0xf0, + 0x2e, 0x58, 0x6e, 0xbb, 0x30, 0xbe, 0xdd, 0x73, 0x5c, 0xe6, 0x78, 0x23, 0x1f, 0x09, 0xd0, 0x37, + 0x14, 0xaa, 0x74, 0x63, 0x89, 0x8a, 0x1a, 0x9e, 0xb4, 0x60, 0xbc, 0x25, 0xcf, 0x51, 0xab, 0x39, + 0xff, 0xf2, 0xf0, 0x1a, 0x8b, 0xc4, 0x91, 0x8c, 0xd4, 0x03, 0x6a, 0x74, 0xe3, 0x33, 0xa0, 0x14, + 0x3e, 0x12, 0x9e, 0xce, 0x68, 0x46, 0xae, 0xa9, 0xbc, 0x11, 0x35, 0x3e, 0x0d, 0xd1, 0x1e, 0xf9, + 0xae, 0x7f, 0x4e, 0xe3, 0x3f, 0x0a, 0x90, 0x56, 0x0b, 0xde, 0xfd, 0x19, 0xd5, 0xce, 0xce, 0xa8, + 0xa5, 0x93, 0x58, 0x80, 0xf9, 0x93, 0xca, 0xf8, 0xab, 0x22, 0x8c, 0xa9, 0xff, 0xa0, 0x39, 0xfd, + 0x34, 0x10, 0x9a, 0x4a, 0x03, 0x59, 0x1c, 0x51, 0x38, 0x0e, 0x4c, 0x02, 0xe9, 0x64, 0x92, 0x40, + 0x46, 0xbd, 0x25, 0xfd, 0x21, 0x29, 0x20, 0x7f, 0x5f, 0x00, 0x25, 0x9a, 0x57, 0xbd, 0x90, 0x99, + 0x9e, 0x25, 0xac, 0x30, 0xb5, 0x0f, 0x8c, 0x1a, 0x6b, 0x54, 0xf1, 0x78, 0xb9, 0xf5, 0x8b, 0xdf, + 0x5a, 0xee, 0x93, 0xf7, 0x43, 0x75, 0xd7, 0x0f, 0x99, 0x90, 0xf5, 0xc5, 0xb4, 0x9b, 0xe5, 0x15, + 0x55, 0x8e, 0x11, 0x45, 0x36, 0x9c, 0x50, 0x19, 0x1c, 0x4e, 0x30, 0xfe, 0xa8, 0x08, 0x13, 0xa9, + 0xbb, 0xf1, 0x87, 0xce, 0x68, 0xc9, 0x24, 0x94, 0x14, 0x4f, 0x3e, 0xa1, 0x24, 0x2f, 0x69, 0xa6, + 0x34, 0x62, 0xd2, 0x4c, 0xf9, 0x38, 0x49, 0x33, 0xc6, 0xd7, 0x0b, 0x00, 0x7a, 0xb4, 0x4e, 0x3d, + 0x9f, 0xc5, 0x4e, 0xe7, 0xb3, 0x8c, 0x3c, 0xaf, 0xf2, 0xb3, 0x59, 0xfe, 0xa2, 0xa2, 0x5f, 0x49, + 0xe4, 0xb2, 0xbc, 0x5d, 0x80, 0x29, 0x33, 0x95, 0x1f, 0x32, 0xb2, 0x7a, 0x99, 0x49, 0x37, 0x89, + 0xfe, 0xa5, 0x26, 0x5d, 0x8e, 0x19, 0xb6, 0xe4, 0x45, 0x98, 0xe8, 0xaa, 0xe0, 0xf9, 0xcd, 0x78, + 0xda, 0x47, 0x67, 0xe3, 0x36, 0x12, 0x75, 0x98, 0xa2, 0x7c, 0x48, 0x3e, 0x4e, 0xe9, 0x44, 0xf2, + 0x71, 0x92, 0x49, 0xff, 0xe5, 0x07, 0x26, 0xfd, 0xef, 0x41, 0x6d, 0x27, 0xf0, 0x3b, 0x22, 0xe5, + 0x45, 0xdd, 0xaf, 0x7e, 0x7d, 0x84, 0x3d, 0x25, 0xfe, 0x67, 0x91, 0x78, 0x6b, 0x5d, 0xd6, 0xf8, + 0x18, 0xb3, 0x12, 0xfe, 0x61, 0x5f, 0x72, 0x1d, 0x3b, 0x49, 0xae, 0x91, 0x2c, 0xd9, 0x94, 0xe8, + 0xa8, 0xd9, 0xa4, 0xd3, 0x5c, 0xc6, 0xdf, 0x9d, 0x34, 0x17, 0xe3, 0x1b, 0x91, 0x00, 0x6b, 0x66, + 0x8e, 0xcf, 0x17, 0x06, 0x1c, 0x9f, 0x57, 0x77, 0xff, 0x24, 0x13, 0x32, 0x9e, 0x85, 0xb1, 0x80, + 0x9a, 0xa1, 0xef, 0xa9, 0xab, 0xc5, 0x22, 0xf1, 0x8f, 0xa2, 0x14, 0x55, 0x6d, 0x32, 0x71, 0xa3, + 0xf8, 0x90, 0xc4, 0x8d, 0xf7, 0x27, 0x26, 0x88, 0xcc, 0xcc, 0x8b, 0xd6, 0x7a, 0xce, 0x24, 0x11, + 0x51, 0x5d, 0xf5, 0xd7, 0x93, 0x95, 0x6c, 0x54, 0x57, 0xfd, 0x2d, 0x64, 0x44, 0x41, 0x6c, 0x98, + 0x70, 0xcd, 0x90, 0x89, 0xe0, 0x88, 0xbd, 0xc0, 0x86, 0xc8, 0x0a, 0x89, 0x96, 0xd1, 0x5a, 0x02, + 0x07, 0x53, 0xa8, 0xc6, 0x41, 0x09, 0x32, 0x66, 0xc8, 0x8f, 0xfc, 0xee, 0xff, 0xa7, 0xfc, 0xee, + 0xef, 0x14, 0x21, 0x5e, 0x53, 0xc7, 0x8c, 0x0d, 0xbf, 0x06, 0xd5, 0x8e, 0x79, 0x6f, 0x89, 0xba, + 0xe6, 0xfe, 0x28, 0x37, 0x52, 0xaf, 0x2b, 0x0c, 0x8c, 0xd0, 0x48, 0x08, 0xe0, 0x44, 0xb7, 0x05, + 0x8d, 0xec, 0xd6, 0x8c, 0x2f, 0x1e, 0x92, 0xee, 0xa1, 0xf8, 0x19, 0x13, 0x6c, 0x8c, 0x83, 0x02, + 0xa8, 0x6b, 0xa5, 0x08, 0x85, 0xca, 0x8e, 0x73, 0x8f, 0xda, 0x23, 0x67, 0x0e, 0x25, 0xae, 0xfd, + 0x97, 0x7e, 0x5b, 0x51, 0x80, 0x12, 0x9d, 0x74, 0x60, 0x3c, 0x94, 0x6e, 0x75, 0x35, 0x7e, 0xc3, + 0x7b, 0x3b, 0x53, 0xee, 0x79, 0x75, 0x49, 0x94, 0x2c, 0x42, 0xcd, 0xa3, 0xf1, 0x89, 0xaf, 0x7d, + 0xfb, 0xf2, 0x63, 0x5f, 0xff, 0xf6, 0xe5, 0xc7, 0xbe, 0xf9, 0xed, 0xcb, 0x8f, 0x7d, 0xf6, 0xf0, + 0x72, 0xe1, 0x6b, 0x87, 0x97, 0x0b, 0x5f, 0x3f, 0xbc, 0x5c, 0xf8, 0xe6, 0xe1, 0xe5, 0xc2, 0xbf, + 0x1c, 0x5e, 0x2e, 0xfc, 0xc6, 0xbf, 0x5e, 0x7e, 0xec, 0xe3, 0x2f, 0x0c, 0xf9, 0xaf, 0xc9, 0xff, + 0x1b, 0x00, 0x00, 0xff, 0xff, 0xc9, 0x40, 0x9d, 0x0c, 0x6f, 0x79, 0x00, 0x00, } func (m *AbstractPodTemplate) Marshal() (dAtA []byte, err error) { @@ -4470,6 +4504,65 @@ func (m *HTTPSource) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *IdleSource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *IdleSource) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *IdleSource) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.IncrementBy != nil { + { + size, err := m.IncrementBy.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + if m.StepInterval != nil { + { + size, err := m.StepInterval.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if m.Threshold != nil { + { + size, err := m.Threshold.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *InterStepBufferService) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -7430,6 +7523,18 @@ func (m *Watermark) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.IdleSource != nil { + { + size, err := m.IdleSource.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } if m.MaxDelay != nil { { size, err := m.MaxDelay.MarshalToSizedBuffer(dAtA[:i]) @@ -8178,6 +8283,27 @@ func (m *HTTPSource) Size() (n int) { return n } +func (m *IdleSource) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Threshold != nil { + l = m.Threshold.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + if m.StepInterval != nil { + l = m.StepInterval.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + if m.IncrementBy != nil { + l = m.IncrementBy.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + return n +} + func (m *InterStepBufferService) Size() (n int) { if m == nil { return 0 @@ -9278,6 +9404,10 @@ func (m *Watermark) Size() (n int) { l = m.MaxDelay.Size() n += 1 + l + sovGenerated(uint64(l)) } + if m.IdleSource != nil { + l = m.IdleSource.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -9789,6 +9919,18 @@ func (this *HTTPSource) String() string { }, "") return s } +func (this *IdleSource) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&IdleSource{`, + `Threshold:` + strings.Replace(fmt.Sprintf("%v", this.Threshold), "Duration", "v11.Duration", 1) + `,`, + `StepInterval:` + strings.Replace(fmt.Sprintf("%v", this.StepInterval), "Duration", "v11.Duration", 1) + `,`, + `IncrementBy:` + strings.Replace(fmt.Sprintf("%v", this.IncrementBy), "Duration", "v11.Duration", 1) + `,`, + `}`, + }, "") + return s +} func (this *InterStepBufferService) String() string { if this == nil { return "nil" @@ -10520,6 +10662,7 @@ func (this *Watermark) String() string { s := strings.Join([]string{`&Watermark{`, `Disabled:` + fmt.Sprintf("%v", this.Disabled) + `,`, `MaxDelay:` + strings.Replace(fmt.Sprintf("%v", this.MaxDelay), "Duration", "v11.Duration", 1) + `,`, + `IdleSource:` + strings.Replace(this.IdleSource.String(), "IdleSource", "IdleSource", 1) + `,`, `}`, }, "") return s @@ -16528,6 +16671,164 @@ func (m *HTTPSource) Unmarshal(dAtA []byte) error { } return nil } +func (m *IdleSource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: IdleSource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: IdleSource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Threshold", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Threshold == nil { + m.Threshold = &v11.Duration{} + } + if err := m.Threshold.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StepInterval", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.StepInterval == nil { + m.StepInterval = &v11.Duration{} + } + if err := m.StepInterval.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field IncrementBy", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.IncrementBy == nil { + m.IncrementBy = &v11.Duration{} + } + if err := m.IncrementBy.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *InterStepBufferService) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -25518,6 +25819,42 @@ func (m *Watermark) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field IdleSource", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.IdleSource == nil { + m.IdleSource = &IdleSource{} + } + if err := m.IdleSource.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/numaflow/v1alpha1/generated.proto b/pkg/apis/numaflow/v1alpha1/generated.proto index ab5582b03b..d4bb9d2363 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.proto +++ b/pkg/apis/numaflow/v1alpha1/generated.proto @@ -521,6 +521,22 @@ message HTTPSource { optional bool service = 2; } +message IdleSource { + // Threshold is the duration after which a source is marked as Idle due to lack of data. + // Ex: If watermark found to be idle after the Threshold duration then the watermark is progressed by `IncrementBy`. + optional k8s.io.apimachinery.pkg.apis.meta.v1.Duration threshold = 1; + + // StepInterval is the duration between the subsequent increment of the watermark as long the source remains Idle. + // The default value is 0s which means that once we detect idle source, we will be incrementing the watermark by + // `IncrementBy` for time we detect that we source is empty (in other words, this will be a very frequent update). + // +kubebuilder:default="0s" + // +optional + optional k8s.io.apimachinery.pkg.apis.meta.v1.Duration stepInterval = 2; + + // IncrementBy is the duration to be added to the current watermark to progress the watermark when source is idling. + optional k8s.io.apimachinery.pkg.apis.meta.v1.Duration incrementBy = 3; +} + // +genclient // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=isbsvc @@ -904,7 +920,7 @@ message PipelineSpec { // +optional optional Watermark watermark = 6; - // Templates is used to customize additional kubernetes resources required for the Pipeline + // Templates are used to customize additional kubernetes resources required for the Pipeline // +optional optional Templates templates = 7; @@ -1351,6 +1367,10 @@ message Watermark { // +kubebuilder:default="0s" // +optional optional k8s.io.apimachinery.pkg.apis.meta.v1.Duration maxDelay = 2; + + // IdleSource defines the idle watermark properties, it could be configured in case source is idling. + // +optional + optional IdleSource idleSource = 3; } // Window describes windowing strategy diff --git a/pkg/apis/numaflow/v1alpha1/openapi_generated.go b/pkg/apis/numaflow/v1alpha1/openapi_generated.go index ed1da40fa2..abd6a45e0c 100644 --- a/pkg/apis/numaflow/v1alpha1/openapi_generated.go +++ b/pkg/apis/numaflow/v1alpha1/openapi_generated.go @@ -55,6 +55,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.GetVertexPodSpecReq": schema_pkg_apis_numaflow_v1alpha1_GetVertexPodSpecReq(ref), "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.GroupBy": schema_pkg_apis_numaflow_v1alpha1_GroupBy(ref), "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.HTTPSource": schema_pkg_apis_numaflow_v1alpha1_HTTPSource(ref), + "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.IdleSource": schema_pkg_apis_numaflow_v1alpha1_IdleSource(ref), "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.InterStepBufferService": schema_pkg_apis_numaflow_v1alpha1_InterStepBufferService(ref), "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.InterStepBufferServiceList": schema_pkg_apis_numaflow_v1alpha1_InterStepBufferServiceList(ref), "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.InterStepBufferServiceSpec": schema_pkg_apis_numaflow_v1alpha1_InterStepBufferServiceSpec(ref), @@ -1777,6 +1778,38 @@ func schema_pkg_apis_numaflow_v1alpha1_HTTPSource(ref common.ReferenceCallback) } } +func schema_pkg_apis_numaflow_v1alpha1_IdleSource(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "threshold": { + SchemaProps: spec.SchemaProps{ + Description: "Threshold is the duration after which a source is marked as Idle due to lack of data. Ex: If watermark found to be idle after the Threshold duration then the watermark is progressed by `IncrementBy`.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + "stepInterval": { + SchemaProps: spec.SchemaProps{ + Description: "StepInterval is the duration between the subsequent increment of the watermark as long the source remains Idle. The default value is 0s which means that once we detect idle source, we will be incrementing the watermark by `IncrementBy` for time we detect that we source is empty (in other words, this will be a very frequent update).", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + "incrementBy": { + SchemaProps: spec.SchemaProps{ + Description: "IncrementBy is the duration to be added to the current watermark to progress the watermark when source is idling.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, + } +} + func schema_pkg_apis_numaflow_v1alpha1_InterStepBufferService(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -3077,7 +3110,7 @@ func schema_pkg_apis_numaflow_v1alpha1_PipelineSpec(ref common.ReferenceCallback }, "templates": { SchemaProps: spec.SchemaProps{ - Description: "Templates is used to customize additional kubernetes resources required for the Pipeline", + Description: "Templates are used to customize additional kubernetes resources required for the Pipeline", Ref: ref("github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.Templates"), }, }, @@ -4758,11 +4791,17 @@ func schema_pkg_apis_numaflow_v1alpha1_Watermark(ref common.ReferenceCallback) c Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, + "idleSource": { + SchemaProps: spec.SchemaProps{ + Description: "IdleSource defines the idle watermark properties, it could be configured in case source is idling.", + Ref: ref("github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.IdleSource"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, + "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1.IdleSource", "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, } } diff --git a/pkg/apis/numaflow/v1alpha1/pipeline_types.go b/pkg/apis/numaflow/v1alpha1/pipeline_types.go index 0be9e43368..ec36c35684 100644 --- a/pkg/apis/numaflow/v1alpha1/pipeline_types.go +++ b/pkg/apis/numaflow/v1alpha1/pipeline_types.go @@ -453,7 +453,7 @@ type PipelineSpec struct { // +kubebuilder:default={"disabled": false} // +optional Watermark Watermark `json:"watermark,omitempty" protobuf:"bytes,6,opt,name=watermark"` - // Templates is used to customize additional kubernetes resources required for the Pipeline + // Templates are used to customize additional kubernetes resources required for the Pipeline // +optional Templates *Templates `json:"templates,omitempty" protobuf:"bytes,7,opt,name=templates"` // SideInputs defines the Side Inputs of a pipeline. @@ -499,6 +499,9 @@ type Watermark struct { // +kubebuilder:default="0s" // +optional MaxDelay *metav1.Duration `json:"maxDelay,omitempty" protobuf:"bytes,2,opt,name=maxDelay"` + // IdleSource defines the idle watermark properties, it could be configured in case source is idling. + // +optional + IdleSource *IdleSource `json:"idleSource,omitempty" protobuf:"bytes,3,opt,name=idleSource"` } // GetMaxDelay returns the configured max delay with a default value. @@ -509,6 +512,43 @@ func (wm Watermark) GetMaxDelay() time.Duration { return time.Duration(0) } +type IdleSource struct { + // Threshold is the duration after which a source is marked as Idle due to lack of data. + // Ex: If watermark found to be idle after the Threshold duration then the watermark is progressed by `IncrementBy`. + Threshold *metav1.Duration `json:"threshold,omitempty" protobuf:"bytes,1,opt,name=threshold"` + // StepInterval is the duration between the subsequent increment of the watermark as long the source remains Idle. + // The default value is 0s which means that once we detect idle source, we will be incrementing the watermark by + // `IncrementBy` for time we detect that we source is empty (in other words, this will be a very frequent update). + // +kubebuilder:default="0s" + // +optional + StepInterval *metav1.Duration `json:"stepInterval,omitempty" protobuf:"bytes,2,opt,name=stepInterval"` + // IncrementBy is the duration to be added to the current watermark to progress the watermark when source is idling. + IncrementBy *metav1.Duration `json:"incrementBy,omitempty" protobuf:"bytes,3,opt,name=incrementBy"` +} + +func (is IdleSource) GetThreshold() time.Duration { + if is.Threshold != nil { + return is.Threshold.Duration + } + + return time.Duration(0) +} + +func (is IdleSource) GetIncrementBy() time.Duration { + if is.IncrementBy != nil { + return is.IncrementBy.Duration + } + + return time.Duration(0) +} + +func (is IdleSource) GetStepInterval() time.Duration { + if is.StepInterval != nil { + return is.StepInterval.Duration + } + return time.Duration(0) +} + type Templates struct { // DaemonTemplate is used to customize the Daemon Deployment. // +optional diff --git a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go index 6c6245fa48..b2e4a16af6 100644 --- a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go @@ -820,6 +820,37 @@ func (in *HTTPSource) DeepCopy() *HTTPSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IdleSource) DeepCopyInto(out *IdleSource) { + *out = *in + if in.Threshold != nil { + in, out := &in.Threshold, &out.Threshold + *out = new(metav1.Duration) + **out = **in + } + if in.StepInterval != nil { + in, out := &in.StepInterval, &out.StepInterval + *out = new(metav1.Duration) + **out = **in + } + if in.IncrementBy != nil { + in, out := &in.IncrementBy, &out.IncrementBy + *out = new(metav1.Duration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IdleSource. +func (in *IdleSource) DeepCopy() *IdleSource { + if in == nil { + return nil + } + out := new(IdleSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InterStepBufferService) DeepCopyInto(out *InterStepBufferService) { *out = *in @@ -2345,6 +2376,11 @@ func (in *Watermark) DeepCopyInto(out *Watermark) { *out = new(metav1.Duration) **out = **in } + if in.IdleSource != nil { + in, out := &in.IdleSource, &out.IdleSource + *out = new(IdleSource) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/daemon/server/daemon_server.go b/pkg/daemon/server/daemon_server.go index 60067e6631..6ba93b6d83 100644 --- a/pkg/daemon/server/daemon_server.go +++ b/pkg/daemon/server/daemon_server.go @@ -149,7 +149,7 @@ func (ds *daemonServer) Run(ctx context.Context) error { func (ds *daemonServer) newGRPCServer( isbSvcClient isbsvc.ISBService, - wmFetchers map[v1alpha1.Edge][]fetch.UXFetcher, + wmFetchers map[v1alpha1.Edge][]fetch.HeadFetcher, rater server.Ratable) (*grpc.Server, error) { // "Prometheus histograms are a great way to measure latency distributions of your RPCs. // However, since it is a bad practice to have metrics of high cardinality the latency monitoring metrics are disabled by default. diff --git a/pkg/daemon/server/service/pipeline_metrics_query.go b/pkg/daemon/server/service/pipeline_metrics_query.go index 5df96c1d98..ee88b76db4 100644 --- a/pkg/daemon/server/service/pipeline_metrics_query.go +++ b/pkg/daemon/server/service/pipeline_metrics_query.go @@ -49,7 +49,7 @@ type pipelineMetadataQuery struct { isbSvcClient isbsvc.ISBService pipeline *v1alpha1.Pipeline httpClient metricsHttpClient - watermarkFetchers map[v1alpha1.Edge][]fetch.UXFetcher + watermarkFetchers map[v1alpha1.Edge][]fetch.HeadFetcher rater rater.Ratable } @@ -63,7 +63,7 @@ const ( func NewPipelineMetadataQuery( isbSvcClient isbsvc.ISBService, pipeline *v1alpha1.Pipeline, - wmFetchers map[v1alpha1.Edge][]fetch.UXFetcher, + wmFetchers map[v1alpha1.Edge][]fetch.HeadFetcher, rater rater.Ratable) (*pipelineMetadataQuery, error) { var err error ps := pipelineMetadataQuery{ diff --git a/pkg/daemon/server/service/pipeline_watermark_query.go b/pkg/daemon/server/service/pipeline_watermark_query.go index 2112b2714b..c86ce8df9e 100644 --- a/pkg/daemon/server/service/pipeline_watermark_query.go +++ b/pkg/daemon/server/service/pipeline_watermark_query.go @@ -31,14 +31,14 @@ import ( // BuildUXEdgeWatermarkFetchers returns a map of the watermark fetchers, where key is the buffer name, // value is a list of fetchers to the buffers. -func BuildUXEdgeWatermarkFetchers(ctx context.Context, pipeline *v1alpha1.Pipeline, wmStores map[v1alpha1.Edge][]store.WatermarkStore) (map[v1alpha1.Edge][]fetch.UXFetcher, error) { - var wmFetchers = make(map[v1alpha1.Edge][]fetch.UXFetcher) +func BuildUXEdgeWatermarkFetchers(ctx context.Context, pipeline *v1alpha1.Pipeline, wmStores map[v1alpha1.Edge][]store.WatermarkStore) (map[v1alpha1.Edge][]fetch.HeadFetcher, error) { + var wmFetchers = make(map[v1alpha1.Edge][]fetch.HeadFetcher) if pipeline.Spec.Watermark.Disabled { return wmFetchers, nil } for edge, stores := range wmStores { - var fetchers []fetch.UXFetcher + var fetchers []fetch.HeadFetcher isReduce := pipeline.GetVertex(edge.To).IsReduceUDF() partitionCount := pipeline.GetVertex(edge.To).GetPartitionCount() for i, s := range stores { diff --git a/pkg/isb/interfaces.go b/pkg/isb/interfaces.go index d6a415e423..6593ec0826 100644 --- a/pkg/isb/interfaces.go +++ b/pkg/isb/interfaces.go @@ -30,12 +30,6 @@ import ( const PendingNotAvailable = int64(math.MinInt64) -// LagReader is the interface that wraps the Pending method. -type LagReader interface { - // Pending returns the pending messages number. - Pending(context.Context) (int64, error) -} - // BufferWriter is the buffer to which we are writing. type BufferWriter interface { BufferWriterInformation @@ -57,8 +51,21 @@ type BufferReader interface { Ack(context.Context, []Offset) []error // NoAck cancels acknowledgement of an array of offset. NoAck(context.Context, []Offset) + // Pending returns the count of pending messages. + Pending(context.Context) (int64, error) } +// LagReader is the interface that wraps the Pending method and GetName method. +// will be used by the metrics server to get the pending messages count. +type LagReader interface { + GetName() string + // Pending returns the pending messages number. + Pending(context.Context) (int64, error) +} + +// BufferReader can be used as LagReader. +var _ LagReader = (BufferReader)(nil) + // BufferReaderInformation has information regarding the buffer we are reading from. type BufferReaderInformation interface { // GetName returns the name. @@ -75,11 +82,6 @@ type BufferWriterInformation interface { GetPartitionIdx() int32 } -// SourceWatermarkPublisher publishes source watermarks based on a list of isb.ReadMessage -type SourceWatermarkPublisher interface { - PublishSourceWatermarks([]*ReadMessage) -} - // Offset is an interface used in the ReadMessage referencing offset information. type Offset interface { // String returns the offset identifier diff --git a/pkg/metrics/metrics_server.go b/pkg/metrics/metrics_server.go index 774db0dc33..bb7626aa78 100644 --- a/pkg/metrics/metrics_server.go +++ b/pkg/metrics/metrics_server.go @@ -107,7 +107,7 @@ func WithHealthCheckExecutor(f func() error) Option { } // NewMetricsOptions returns a metrics option list. -func NewMetricsOptions(ctx context.Context, vertex *dfv1.Vertex, healthCheckers []HealthChecker, readers []isb.BufferReader) []Option { +func NewMetricsOptions(ctx context.Context, vertex *dfv1.Vertex, healthCheckers []HealthChecker, readers []isb.LagReader) []Option { metricsOpts := []Option{ WithLookbackSeconds(int64(vertex.Spec.Scale.GetLookbackSeconds())), } @@ -124,9 +124,7 @@ func NewMetricsOptions(ctx context.Context, vertex *dfv1.Vertex, healthCheckers lagReaders := make(map[string]isb.LagReader) for _, reader := range readers { - if x, ok := reader.(isb.LagReader); ok { - lagReaders[reader.GetName()] = x - } + lagReaders[reader.GetName()] = reader } if len(lagReaders) > 0 { metricsOpts = append(metricsOpts, WithLagReaders(lagReaders)) diff --git a/pkg/reconciler/pipeline/validate.go b/pkg/reconciler/pipeline/validate.go index 29b014cbe7..0fb558c022 100644 --- a/pkg/reconciler/pipeline/validate.go +++ b/pkg/reconciler/pipeline/validate.go @@ -199,6 +199,29 @@ func ValidatePipeline(pl *dfv1.Pipeline) error { return err } + if err := validateIdleSource(*pl); err != nil { + return err + } + + return nil +} + +// validateIdleSource validates the idle source watermark config. +// The threshold should be greater than or equal to incrementBy. +func validateIdleSource(pl dfv1.Pipeline) error { + if pl.Spec.Watermark.IdleSource != nil { + if pl.Spec.Watermark.IdleSource.Threshold == nil { + return fmt.Errorf("invalid idle source watermark config, threshold is missing") + } else if pl.Spec.Watermark.IdleSource.Threshold.Duration <= 0 { + return fmt.Errorf("invalid idle source watermark config, threshold should be greater than 0") + } else if pl.Spec.Watermark.IdleSource.IncrementBy == nil { + return fmt.Errorf("invalid idle source watermark config, incrementBy is missing") + } else if pl.Spec.Watermark.IdleSource.IncrementBy.Duration <= 0 { + return fmt.Errorf("invalid idle source watermark config, incrementBy should be greater than 0") + } else if pl.Spec.Watermark.IdleSource.Threshold.Duration < pl.Spec.Watermark.IdleSource.IncrementBy.Duration { + return fmt.Errorf("invalid idle source watermark config, threshold should be greater than or equal to incrementBy") + } + } return nil } diff --git a/pkg/reconciler/pipeline/validate_test.go b/pkg/reconciler/pipeline/validate_test.go index 64d170b240..f84e5292d0 100644 --- a/pkg/reconciler/pipeline/validate_test.go +++ b/pkg/reconciler/pipeline/validate_test.go @@ -955,3 +955,51 @@ func Test_validateCycles(t *testing.T) { }) } } + +func Test_validateIdleSource(t *testing.T) { + testObj := testPipeline.DeepCopy() + testObj.Spec.Watermark.IdleSource = &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 5 * time.Second}, + IncrementBy: &metav1.Duration{Duration: 5 * time.Second}, + } + err := validateIdleSource(*testObj) + assert.NoError(t, err) + + testObj.Spec.Watermark.IdleSource = &dfv1.IdleSource{ + Threshold: nil, + } + err = validateIdleSource(*testObj) + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid idle source watermark config, threshold is missing`) + + testObj.Spec.Watermark.IdleSource = &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 0 * time.Second}, + } + err = validateIdleSource(*testObj) + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid idle source watermark config, threshold should be greater than 0`) + + testObj.Spec.Watermark.IdleSource = &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 5 * time.Second}, + IncrementBy: nil, + } + err = validateIdleSource(*testObj) + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid idle source watermark config, incrementBy is missing`) + + testObj.Spec.Watermark.IdleSource = &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 5 * time.Second}, + IncrementBy: &metav1.Duration{Duration: 0 * time.Second}, + } + err = validateIdleSource(*testObj) + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid idle source watermark config, incrementBy should be greater than 0`) + + testObj.Spec.Watermark.IdleSource = &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 2 * time.Second}, + IncrementBy: &metav1.Duration{Duration: 5 * time.Second}, + } + err = validateIdleSource(*testObj) + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid idle source watermark config, threshold should be greater than or equal to incrementBy`) +} diff --git a/pkg/shared/idlehandler/idlehandler.go b/pkg/shared/idlehandler/idlehandler.go index 237ddbe835..29a0f9247e 100644 --- a/pkg/shared/idlehandler/idlehandler.go +++ b/pkg/shared/idlehandler/idlehandler.go @@ -28,7 +28,6 @@ import ( ) // PublishIdleWatermark publishes a ctrl message with isb.Kind set to WMB. We only send one ctrl message when - func PublishIdleWatermark(ctx context.Context, toBufferPartition isb.BufferWriter, wmPublisher publish.Publisher, idleManager wmb.IdleManager, logger *zap.SugaredLogger, vertexType dfv1.VertexType, wm wmb.Watermark) { var toPartitionName = toBufferPartition.GetName() diff --git a/pkg/shared/idlehandler/source_idlehandler.go b/pkg/shared/idlehandler/source_idlehandler.go new file mode 100644 index 0000000000..f7d34383f3 --- /dev/null +++ b/pkg/shared/idlehandler/source_idlehandler.go @@ -0,0 +1,95 @@ +package idlehandler + +import ( + "time" + + dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" + "github.com/numaproj/numaflow/pkg/watermark/fetch" + "github.com/numaproj/numaflow/pkg/watermark/publish" +) + +// Source Idle Handler has to resolve the following conundrums: +// How to decide if the source is idling? +// If data forwarder is not reading any messages for WatermarkConfig.Threshold (provided by user) +// time, then it is idling. +// When to publish the idle watermark? +// If the source is idling and the step interval has passed (also provided by the user). +// What to publish as the idle watermark? +// The current watermark + WatermarkConfig.IncrementBy (provided by user). We will ensure that the +// increment will never cross (time.Now() - maxDelay). + +// SourceIdleHandler handles operations related to idle watermarks for source. +type SourceIdleHandler struct { + config *dfv1.Watermark + lastIdleWmPublishedTime time.Time + updatedTS time.Time + wmFetcher fetch.SourceFetcher + srcPublisher publish.SourcePublisher +} + +// NewSourceIdleHandler creates a new instance of SrcIdleHandler. +func NewSourceIdleHandler(config *dfv1.Watermark, fetcher fetch.SourceFetcher, publisher publish.SourcePublisher) *SourceIdleHandler { + return &SourceIdleHandler{ + config: config, + wmFetcher: fetcher, + srcPublisher: publisher, + updatedTS: time.Now(), + lastIdleWmPublishedTime: time.UnixMilli(-1), + } +} + +// IsSourceIdling will return true if source has been idling and the step interval has passed. +func (iw *SourceIdleHandler) IsSourceIdling() bool { + return iw.isSourceIdling() && iw.hasStepIntervalPassed() +} + +// isSourceIdling checks if the source is idling by comparing the last updated timestamp with the threshold. +func (iw *SourceIdleHandler) isSourceIdling() bool { + // if the source is not configured for idling, return false + if iw.config == nil || iw.config.IdleSource == nil { + return false + } + + // if the threshold has not passed, return false + if time.Since(iw.updatedTS) < iw.config.IdleSource.GetThreshold() { + return false + } + + return true +} + +// hasStepIntervalPassed verifies if the step interval has passed. +func (iw *SourceIdleHandler) hasStepIntervalPassed() bool { + + // if the last idle watermark published time is -1, it means that the idle watermark has not been published yet. + // -1 is used as the default value for lastIdleWmPublishedTime, so that we immediately publish the idle watermark + // when the source is idling for the first time after the threshold has passed and next subsequent idle watermark + // is published after the step interval has passed. + if iw.lastIdleWmPublishedTime == time.UnixMilli(-1) { + return true + } + + // else make sure duration has passed + return time.Since(iw.lastIdleWmPublishedTime) >= iw.config.IdleSource.GetStepInterval() +} + +// PublishSourceIdleWatermark publishes an idle watermark. +func (iw *SourceIdleHandler) PublishSourceIdleWatermark(partitions []int32) { + // publish the idle watermark, the idle watermark is the current watermark + the increment by value. + nextIdleWM := iw.wmFetcher.ComputeWatermark().Add(iw.config.IdleSource.GetIncrementBy()) + currentTime := time.Now().Add(-1 * iw.config.GetMaxDelay()) + + // if the next idle watermark is after the current time, then set the next idle watermark to the current time. + if nextIdleWM.After(currentTime) { + nextIdleWM = currentTime + } + + iw.srcPublisher.PublishIdleWatermarks(nextIdleWM, partitions) + iw.lastIdleWmPublishedTime = time.Now() +} + +// Reset resets the updatedTS to the current time. +func (iw *SourceIdleHandler) Reset() { + iw.updatedTS = time.Now() + iw.lastIdleWmPublishedTime = time.UnixMilli(-1) +} diff --git a/pkg/shared/idlehandler/source_idlehandler_test.go b/pkg/shared/idlehandler/source_idlehandler_test.go new file mode 100644 index 0000000000..271361fa1e --- /dev/null +++ b/pkg/shared/idlehandler/source_idlehandler_test.go @@ -0,0 +1,96 @@ +package idlehandler + +import ( + dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" + "time" +) + +func TestSourceIdleHandler_IsSourceIdling(t *testing.T) { + type fields struct { + config *dfv1.Watermark + lastIdleWmPublishedTime time.Time + updatedTS time.Time + } + tests := []struct { + name string + fields fields + sleep time.Duration + want bool + }{ + { + name: "Source is not idling as threshold has not passed", + fields: fields{ + config: &dfv1.Watermark{ + IdleSource: &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 5 * time.Second}, + StepInterval: &metav1.Duration{Duration: 2 * time.Second}, + IncrementBy: &metav1.Duration{Duration: 3 * time.Second}, + }, + }, + lastIdleWmPublishedTime: time.UnixMilli(-1), + updatedTS: time.Now(), + }, + sleep: 0 * time.Second, + want: false, + }, + { + name: "Source is idling as threshold has passed", + fields: fields{ + config: &dfv1.Watermark{ + IdleSource: &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 5 * time.Second}, + StepInterval: &metav1.Duration{Duration: 2 * time.Second}, + IncrementBy: &metav1.Duration{Duration: 3 * time.Second}, + }, + }, + lastIdleWmPublishedTime: time.UnixMilli(-1), + updatedTS: time.Now(), + }, + sleep: 5 * time.Second, + want: true, + }, + { + name: "Source is idling, threshold and step interval has passed", + fields: fields{ + config: &dfv1.Watermark{ + IdleSource: &dfv1.IdleSource{ + Threshold: &metav1.Duration{Duration: 5 * time.Second}, + StepInterval: &metav1.Duration{Duration: 2 * time.Second}, + IncrementBy: &metav1.Duration{Duration: 3 * time.Second}, + }, + }, + lastIdleWmPublishedTime: time.Now(), + updatedTS: time.Now(), + }, + sleep: 5 * time.Second, + want: true, + }, + { + name: "Watermark is not enabled for source", + fields: fields{ + config: &dfv1.Watermark{ + IdleSource: nil, + }, + lastIdleWmPublishedTime: time.UnixMilli(-1), + updatedTS: time.Now(), + }, + sleep: 0 * time.Second, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + iw := &SourceIdleHandler{ + config: tt.fields.config, + lastIdleWmPublishedTime: tt.fields.lastIdleWmPublishedTime, + updatedTS: tt.fields.updatedTS, + } + time.Sleep(tt.sleep) + if got := iw.IsSourceIdling(); got != tt.want { + t.Errorf("IsSourceIdling() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/sinks/sink.go b/pkg/sinks/sink.go index 2fff6c890c..afe9d6626a 100644 --- a/pkg/sinks/sink.go +++ b/pkg/sinks/sink.go @@ -191,8 +191,14 @@ func (u *SinkProcessor) Start(ctx context.Context) error { log.Infow("Exited for partition...", zap.String("fromPartition", fromBufferPartitionName)) }(sinker, readers[index].GetName()) } + + // create lag readers from buffer readers + var lagReaders []isb.LagReader + for _, reader := range readers { + lagReaders = append(lagReaders, reader) + } // start metrics server and pass the sinkHandler to it, so that it can be used to check the readiness of the sink - metricsOpts := metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{sinkHandler}, readers) + metricsOpts := metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{sinkHandler}, lagReaders) ms := metrics.NewMetricsServer(u.VertexInstance.Vertex, metricsOpts...) if shutdown, err := ms.Start(ctx); err != nil { return fmt.Errorf("failed to start metrics server, error: %w", err) diff --git a/pkg/sources/forward/data_forward.go b/pkg/sources/forward/data_forward.go index 010dfd3b4e..ccc350d09a 100644 --- a/pkg/sources/forward/data_forward.go +++ b/pkg/sources/forward/data_forward.go @@ -33,6 +33,7 @@ import ( "github.com/numaproj/numaflow/pkg/shared/idlehandler" "github.com/numaproj/numaflow/pkg/shared/logging" "github.com/numaproj/numaflow/pkg/sources/forward/applier" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/watermark/entity" "github.com/numaproj/numaflow/pkg/watermark/fetch" "github.com/numaproj/numaflow/pkg/watermark/publish" @@ -46,43 +47,40 @@ type DataForward struct { ctx context.Context // cancelFn cancels our new context, our cancellation is a little more complex and needs to be well orchestrated, hence // we need something more than a cancel(). - cancelFn context.CancelFunc - // reader reads data from source. - reader isb.BufferReader - // toBuffers store the toVertex name to its owned buffers mapping. - toBuffers map[string][]isb.BufferWriter - toWhichStepDecider forwarder.ToWhichStepDecider - transformer applier.SourceTransformApplier - wmFetcher fetch.Fetcher - toVertexWMStores map[string]store.WatermarkStore - // toVertexWMPublishers stores the toVertex to publisher mapping. - toVertexWMPublishers map[string]map[int32]publish.Publisher - // srcWMPublisher is used to publish source watermark. - srcWMPublisher isb.SourceWatermarkPublisher - opts options - vertexName string - pipelineName string - vertexReplica int32 - // idleManager manages the idle watermark status. - idleManager wmb.IdleManager + cancelFn context.CancelFunc + reader sourcer.SourceReader // reader reads data from source. + toBuffers map[string][]isb.BufferWriter // toBuffers store the toVertex name to its owned buffers mapping. + toWhichStepDecider forwarder.ToWhichStepDecider + transformer applier.SourceTransformApplier + wmFetcher fetch.SourceFetcher + toVertexWMStores map[string]store.WatermarkStore + toVertexWMPublishers map[string]map[int32]publish.Publisher // toVertexWMPublishers stores the toVertex to publisher mapping. + srcWMPublisher publish.SourcePublisher // srcWMPublisher is used to publish source watermark. + opts options + vertexName string + pipelineName string + vertexReplica int32 + watermarkConfig dfv1.Watermark + idleManager wmb.IdleManager // idleManager manages the idle watermark status. + srcIdleHandler *idlehandler.SourceIdleHandler Shutdown } // NewDataForward creates a source data forwarder func NewDataForward( vertexInstance *dfv1.VertexInstance, - fromStep isb.BufferReader, + reader sourcer.SourceReader, toSteps map[string][]isb.BufferWriter, toWhichStepDecider forwarder.ToWhichStepDecider, transformer applier.SourceTransformApplier, - fetchWatermark fetch.Fetcher, - srcWMPublisher isb.SourceWatermarkPublisher, + fetchWatermark fetch.SourceFetcher, + srcWMPublisher publish.SourcePublisher, toVertexWmStores map[string]store.WatermarkStore, idleManager wmb.IdleManager, opts ...Option) (*DataForward, error) { - options := DefaultOptions() + defaultOptions := DefaultOptions() for _, o := range opts { - if err := o(options); err != nil { + if err := o(defaultOptions); err != nil { return nil, err } } @@ -92,12 +90,15 @@ func NewDataForward( toVertexWMPublishers[k] = make(map[int32]publish.Publisher) } + // create a source idle handler + srcIdleHandler := idlehandler.NewSourceIdleHandler(&vertexInstance.Vertex.Spec.Watermark, fetchWatermark, srcWMPublisher) + // creating a context here which is managed by the forwarder's lifecycle ctx, cancel := context.WithCancel(context.Background()) var isdf = DataForward{ ctx: ctx, cancelFn: cancel, - reader: fromStep, + reader: reader, toBuffers: toSteps, toWhichStepDecider: toWhichStepDecider, transformer: transformer, @@ -108,20 +109,23 @@ func NewDataForward( vertexName: vertexInstance.Vertex.Spec.Name, pipelineName: vertexInstance.Vertex.Spec.PipelineName, vertexReplica: vertexInstance.Replica, + watermarkConfig: vertexInstance.Vertex.Spec.Watermark, idleManager: idleManager, + srcIdleHandler: srcIdleHandler, Shutdown: Shutdown{ rwlock: new(sync.RWMutex), }, - opts: *options, + opts: *defaultOptions, } // add logger from parent ctx to child context. - isdf.ctx = logging.WithLogger(ctx, options.logger) + isdf.ctx = logging.WithLogger(ctx, defaultOptions.logger) + // set the current watermark return &isdf, nil } // Start starts reading from source and forwards to the next buffers. Call `Stop` to stop. -func (isdf *DataForward) Start() <-chan struct{} { - log := logging.FromContext(isdf.ctx) +func (df *DataForward) Start() <-chan struct{} { + log := logging.FromContext(df.ctx) stopped := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) @@ -131,8 +135,8 @@ func (isdf *DataForward) Start() <-chan struct{} { defer wg.Done() for { select { - case <-isdf.ctx.Done(): - ok, err := isdf.IsShuttingDown() + case <-df.ctx.Done(): + ok, err := df.IsShuttingDown() if err != nil { // ignore the error for now. log.Errorw("Failed to check if it can shutdown", zap.Error(err)) @@ -146,19 +150,19 @@ func (isdf *DataForward) Start() <-chan struct{} { // shutdown the reader should be empty. } // keep doing what you are good at - isdf.forwardAChunk(isdf.ctx) + df.forwardAChunk(df.ctx) } }() go func() { wg.Wait() // clean up resources for source reader and all the writers if any. - if err := isdf.reader.Close(); err != nil { + if err := df.reader.Close(); err != nil { log.Errorw("Failed to close source reader, shutdown anyways...", zap.Error(err)) } else { - log.Infow("Closed source reader", zap.String("sourceFrom", isdf.reader.GetName())) + log.Infow("Closed source reader", zap.String("sourceFrom", df.reader.GetName())) } - for _, buffer := range isdf.toBuffers { + for _, buffer := range df.toBuffers { for _, partition := range buffer { if err := partition.Close(); err != nil { log.Errorw("Failed to close partition writer, shutdown anyways...", zap.Error(err), zap.String("bufferTo", partition.GetName())) @@ -169,7 +173,7 @@ func (isdf *DataForward) Start() <-chan struct{} { } // the publisher was created by the forwarder, so it should be closed by the forwarder. - for _, toVertexPublishers := range isdf.toVertexWMPublishers { + for _, toVertexPublishers := range df.toVertexWMPublishers { for _, pub := range toVertexPublishers { if err := pub.Close(); err != nil { log.Errorw("Failed to close publisher, shutdown anyways...", zap.Error(err)) @@ -193,24 +197,61 @@ type readWriteMessagePair struct { // for a chunk of messages returned by the first Read call. It will return only if only we are successfully able to ack // the message after forwarding, barring any platform errors. The platform errors include buffer-full, // buffer-not-reachable, etc., but do not include errors due to user code transformer, WhereTo, etc. -func (isdf *DataForward) forwardAChunk(ctx context.Context) { +func (df *DataForward) forwardAChunk(ctx context.Context) { start := time.Now() // There is a chance that we have read the message and the container got forcefully terminated before processing. To provide // at-least-once semantics for reading, during the restart we will have to reprocess all unacknowledged messages. It is the // responsibility of the Read function to do that. - readMessages, err := isdf.reader.Read(ctx, isdf.opts.readBatchSize) + readMessages, err := df.reader.Read(ctx, df.opts.readBatchSize) if err != nil { - isdf.opts.logger.Warnw("failed to read from source", zap.Error(err)) - metrics.ReadMessagesError.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Inc() + df.opts.logger.Warnw("failed to read from source", zap.Error(err)) + metrics.ReadMessagesError.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Inc() } - // Process only if we have any read messages. - // There is a natural looping here if there is an internal error while reading, and we are not able to proceed. + // if there are no read messages, we return early. if len(readMessages) == 0 { + // not idling, so nothing much to do + if !df.srcIdleHandler.IsSourceIdling() { + return + } + + // if the source is idling, we will publish idle watermark to the source and all the toBuffers + // we will not publish idle watermark if the source is not idling. + // publish idle watermark for the source + df.srcIdleHandler.PublishSourceIdleWatermark(df.reader.Partitions()) + + // if we have published idle watermark to source, we need to publish idle watermark to all the toBuffers + // it might not get the latest watermark because of publishing delay, but we will get in the subsequent + // iterations. + + // publish idle watermark for all the toBuffers + fetchedWm := df.wmFetcher.ComputeWatermark() + for toVertexName, toVertexBuffers := range df.toBuffers { + for index := range toVertexBuffers { + // publish idle watermark to all the source partitions owned by this reader. + // it is 1:1 for many (HTTP, tickgen, etc.) but for e.g., for Kafka it is 1:N and the list of partitions in the N could keep changing. + for _, sp := range df.reader.Partitions() { + if vertexPublishers, ok := df.toVertexWMPublishers[toVertexName]; ok { + var publisher, ok = vertexPublishers[sp] + if !ok { + publisher = df.createToVertexWatermarkPublisher(toVertexName, sp) + vertexPublishers[sp] = publisher + } + idlehandler.PublishIdleWatermark(ctx, df.toBuffers[toVertexName][index], publisher, df.idleManager, df.opts.logger, dfv1.VertexTypeSource, fetchedWm) + } + } + } + } + + // len(readMessages) == 0, so we do not have anything more to do return } - metrics.ReadDataMessagesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Add(float64(len(readMessages))) - metrics.ReadMessagesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Add(float64(len(readMessages))) + + // reset the idle handler because we have read messages + df.srcIdleHandler.Reset() + + metrics.ReadDataMessagesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Add(float64(len(readMessages))) + metrics.ReadMessagesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Add(float64(len(readMessages))) // store the offsets of the messages we read from source var readOffsets = make([]isb.Offset, len(readMessages)) @@ -225,9 +266,9 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { var writeOffsets map[string][][]isb.Offset // create space for writeMessages specific to each step as we could forward to all the steps too. var messageToStep = make(map[string][][]isb.Message) - for toVertex := range isdf.toBuffers { + for toVertex := range df.toBuffers { // over allocating to have a predictable pattern - messageToStep[toVertex] = make([][]isb.Message, len(isdf.toBuffers[toVertex])) + messageToStep[toVertex] = make([][]isb.Message, len(df.toBuffers[toVertex])) } // user defined transformer concurrent processing request channel @@ -240,11 +281,11 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { // create a pool of Transformer Processors var wg sync.WaitGroup - for i := 0; i < isdf.opts.transformerConcurrency; i++ { + for i := 0; i < df.opts.transformerConcurrency; i++ { wg.Add(1) go func() { defer wg.Done() - isdf.concurrentApplyTransformer(ctx, transformerCh) + df.concurrentApplyTransformer(ctx, transformerCh) }() } concurrentTransformerProcessingStart := time.Now() @@ -252,7 +293,7 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { // send to transformer only the data messages for idx, m := range readMessages { // emit message size metric - metrics.ReadBytesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Add(float64(len(m.Payload))) + metrics.ReadBytesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Add(float64(len(m.Payload))) // assign watermark to the message m.Watermark = time.Time(processorWM) // send transformer processing work to the channel @@ -264,8 +305,8 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { // wait till the processing is done. this will not be an infinite wait because the transformer processing will exit if // context.Done() is closed. wg.Wait() - isdf.opts.logger.Debugw("concurrent applyTransformer completed", zap.Int("concurrency", isdf.opts.transformerConcurrency), zap.Duration("took", time.Since(concurrentTransformerProcessingStart))) - metrics.SourceTransformerConcurrentProcessingTime.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Observe(float64(time.Since(concurrentTransformerProcessingStart).Microseconds())) + df.opts.logger.Debugw("concurrent applyTransformer completed", zap.Int("concurrency", df.opts.transformerConcurrency), zap.Duration("took", time.Since(concurrentTransformerProcessingStart))) + metrics.SourceTransformerConcurrentProcessingTime.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Observe(float64(time.Since(concurrentTransformerProcessingStart).Microseconds())) // transformer processing is done. // publish source watermark and assign IsLate attribute based on new event time. @@ -282,10 +323,11 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { } } // publish source watermark - isdf.srcWMPublisher.PublishSourceWatermarks(transformedReadMessages) + df.srcWMPublisher.PublishSourceWatermarks(transformedReadMessages) + // update the watermark configs for lastTimestampSrcWMUpdated, lastFetchedSrcWatermark and lastTimestampIdleWMFound. // fetch the source watermark again, we might not get the latest watermark because of publishing delay, // but ideally we should use the latest to determine the IsLate attribute. - processorWM = isdf.wmFetcher.ComputeWatermark(readMessages[0].ReadOffset, isdf.reader.GetPartitionIdx()) + processorWM = df.wmFetcher.ComputeWatermark() // assign isLate for _, m := range writeMessages { if processorWM.After(m.EventTime) { // Set late data at source level @@ -300,14 +342,14 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { // Look for errors in transformer processing if we see even 1 error we return. // Handling partial retrying is not worth ATM. if m.transformerError != nil { - metrics.SourceTransformerError.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Inc() - isdf.opts.logger.Errorw("failed to apply source transformer", zap.Error(m.transformerError)) + metrics.SourceTransformerError.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Inc() + df.opts.logger.Errorw("failed to apply source transformer", zap.Error(m.transformerError)) return } // update toBuffers for _, message := range m.writeMessages { - if err := isdf.whereToStep(message, messageToStep, m.readMessage); err != nil { - isdf.opts.logger.Errorw("failed in whereToStep", zap.Error(err)) + if err := df.whereToStep(message, messageToStep, m.readMessage); err != nil { + df.opts.logger.Errorw("failed in whereToStep", zap.Error(err)) return } } @@ -316,9 +358,9 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { } // forward the messages to the edge buffer (could be multiple edges) - writeOffsets, err = isdf.writeToBuffers(ctx, messageToStep) + writeOffsets, err = df.writeToBuffers(ctx, messageToStep) if err != nil { - isdf.opts.logger.Errorw("failed to write to toBuffers", zap.Error(err)) + df.opts.logger.Errorw("failed to write to toBuffers", zap.Error(err)) return } @@ -326,12 +368,13 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { // a watermark in this batch processing cycle. // It's used to determine which buffers should receive an idle watermark. // It is created as a slice because it tracks per partition activity info. + // when there are no messages read from the source, we will publish idle watermark to all the toBuffers. var activeWatermarkBuffers = make(map[string][]bool) // forward the highest watermark to all the edges to avoid idle edge problem // TODO: sort and get the highest value for toVertexName, toVertexBufferOffsets := range writeOffsets { activeWatermarkBuffers[toVertexName] = make([]bool, len(toVertexBufferOffsets)) - if vertexPublishers, ok := isdf.toVertexWMPublishers[toVertexName]; ok { + if vertexPublishers, ok := df.toVertexWMPublishers[toVertexName]; ok { for index, offsets := range toVertexBufferOffsets { if len(offsets) > 0 { // publish watermark to all the source partitions @@ -339,14 +382,15 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { for sp := range sourcePartitionsIndices { var publisher, ok = vertexPublishers[sp] if !ok { - publisher = isdf.createToVertexWatermarkPublisher(toVertexName, sp) + publisher = df.createToVertexWatermarkPublisher(toVertexName, sp) vertexPublishers[sp] = publisher } publisher.PublishWatermark(processorWM, offsets[len(offsets)-1], int32(index)) activeWatermarkBuffers[toVertexName][index] = true + // update the watermark configs for lastTimestampSrcWMUpdated, lastFetchedSrcWatermark and lastTimestampIdleWMFound. // reset because the toBuffer partition is no longer idling - isdf.idleManager.Reset(isdf.toBuffers[toVertexName][index].GetName()) + df.idleManager.Reset(df.toBuffers[toVertexName][index].GetName()) } } // This (len(offsets) == 0) happens at conditional forwarding, there's no data written to the buffer @@ -354,21 +398,20 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { } } - // condition "len(activeWatermarkBuffers) < len(isdf.toVertexWMPublishers)": + // condition "len(activeWatermarkBuffers) < len(df.toVertexWMPublishers)": // send idle watermark only if we have idled out buffers - for toVertexName := range isdf.toVertexWMPublishers { + for toVertexName := range df.toVertexWMPublishers { for index, activePartition := range activeWatermarkBuffers[toVertexName] { if !activePartition { - // use the watermark of the current read batch for the idle watermark // same as read len==0 because there's no event published to the buffer - if vertexPublishers, ok := isdf.toVertexWMPublishers[toVertexName]; ok { + if vertexPublishers, ok := df.toVertexWMPublishers[toVertexName]; ok { for sp := range sourcePartitionsIndices { var publisher, ok = vertexPublishers[sp] if !ok { - publisher = isdf.createToVertexWatermarkPublisher(toVertexName, sp) + publisher = df.createToVertexWatermarkPublisher(toVertexName, sp) vertexPublishers[sp] = publisher } - idlehandler.PublishIdleWatermark(ctx, isdf.toBuffers[toVertexName][index], publisher, isdf.idleManager, isdf.opts.logger, dfv1.VertexTypeSource, processorWM) + idlehandler.PublishIdleWatermark(ctx, df.toBuffers[toVertexName][index], publisher, df.idleManager, df.opts.logger, dfv1.VertexTypeSource, processorWM) } } } @@ -377,29 +420,29 @@ func (isdf *DataForward) forwardAChunk(ctx context.Context) { // when we apply transformer, we don't handle partial errors (it's either non or all, non will return early), // so we should be able to ack all the readOffsets including data messages and control messages - err = isdf.ackFromSource(ctx, readOffsets) + err = df.ackFromSource(ctx, readOffsets) // implicit return for posterity :-) if err != nil { - isdf.opts.logger.Errorw("failed to ack from source", zap.Error(err)) - metrics.AckMessageError.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Add(float64(len(readOffsets))) + df.opts.logger.Errorw("failed to ack from source", zap.Error(err)) + metrics.AckMessageError.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Add(float64(len(readOffsets))) return } - metrics.AckMessagesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Add(float64(len(readOffsets))) + metrics.AckMessagesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Add(float64(len(readOffsets))) // ProcessingTimes of the entire forwardAChunk - metrics.ForwardAChunkProcessingTime.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica))}).Observe(float64(time.Since(start).Microseconds())) + metrics.ForwardAChunkProcessingTime.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica))}).Observe(float64(time.Since(start).Microseconds())) } -func (isdf *DataForward) ackFromSource(ctx context.Context, offsets []isb.Offset) error { +func (df *DataForward) ackFromSource(ctx context.Context, offsets []isb.Offset) error { // for all the sources, we either ack all offsets or none. // when a batch ack fails, the source Ack() function populate the error array with the same error; // hence we can just return the first error. - return isdf.reader.Ack(ctx, offsets)[0] + return df.reader.Ack(ctx, offsets)[0] } // writeToBuffers is a blocking call until all the messages have been forwarded to all the toBuffers, or a shutdown // has been initiated while we are stuck looping on an InternalError. -func (isdf *DataForward) writeToBuffers( +func (df *DataForward) writeToBuffers( ctx context.Context, messageToStep map[string][][]isb.Message, ) (writeOffsets map[string][][]isb.Offset, err error) { // messageToStep contains all the to buffers, so the messages could be empty (conditional forwarding). @@ -408,9 +451,9 @@ func (isdf *DataForward) writeToBuffers( for toVertexName, toVertexMessages := range messageToStep { writeOffsets[toVertexName] = make([][]isb.Offset, len(toVertexMessages)) } - for toVertexName, toVertexBuffer := range isdf.toBuffers { + for toVertexName, toVertexBuffer := range df.toBuffers { for index, partition := range toVertexBuffer { - writeOffsets[toVertexName][index], err = isdf.writeToBuffer(ctx, partition, messageToStep[toVertexName][index]) + writeOffsets[toVertexName][index], err = df.writeToBuffer(ctx, partition, messageToStep[toVertexName][index]) if err != nil { return nil, err } @@ -420,7 +463,7 @@ func (isdf *DataForward) writeToBuffers( } // writeToBuffer forwards an array of messages to a single buffer and is a blocking call or until shutdown has been initiated. -func (isdf *DataForward) writeToBuffer(ctx context.Context, toBufferPartition isb.BufferWriter, messages []isb.Message) (writeOffsets []isb.Offset, err error) { +func (df *DataForward) writeToBuffer(ctx context.Context, toBufferPartition isb.BufferWriter, messages []isb.Message) (writeOffsets []isb.Offset, err error) { var ( totalCount int writeCount int @@ -445,10 +488,10 @@ func (isdf *DataForward) writeToBuffer(ctx context.Context, toBufferPartition is needRetry = true // we retry only failed messages failedMessages = append(failedMessages, msg) - metrics.WriteMessagesError.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Inc() + metrics.WriteMessagesError.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Inc() // a shutdown can break the blocking loop caused due to InternalErr - if ok, _ := isdf.IsShuttingDown(); ok { - metrics.PlatformError.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica))}).Inc() + if ok, _ := df.IsShuttingDown(); ok { + metrics.PlatformError.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica))}).Inc() return writeOffsets, fmt.Errorf("writeToBuffer failed, Stop called while stuck on an internal error with failed messages:%d, %v", len(failedMessages), errs) } } @@ -463,63 +506,63 @@ func (isdf *DataForward) writeToBuffer(ctx context.Context, toBufferPartition is } if needRetry { - isdf.opts.logger.Errorw("Retrying failed messages", + df.opts.logger.Errorw("Retrying failed messages", zap.Any("errors", errorArrayToMap(errs)), - zap.String(metrics.LabelPipeline, isdf.pipelineName), - zap.String(metrics.LabelVertex, isdf.vertexName), + zap.String(metrics.LabelPipeline, df.pipelineName), + zap.String(metrics.LabelVertex, df.vertexName), zap.String(metrics.LabelPartitionName, toBufferPartition.GetName()), ) // set messages to the failed slice for the retry messages = failedMessages // TODO: implement retry with backoff etc. - time.Sleep(isdf.opts.retryInterval) + time.Sleep(df.opts.retryInterval) } else { break } } - metrics.DropMessagesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(float64(totalCount - writeCount)) - metrics.DropBytesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(dropBytes) - metrics.WriteMessagesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(float64(writeCount)) - metrics.WriteBytesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(writeBytes) + metrics.DropMessagesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(float64(totalCount - writeCount)) + metrics.DropBytesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(dropBytes) + metrics.WriteMessagesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(float64(writeCount)) + metrics.WriteBytesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: toBufferPartition.GetName()}).Add(writeBytes) return writeOffsets, nil } // concurrentApplyTransformer applies the transformer based on the request from the channel -func (isdf *DataForward) concurrentApplyTransformer(ctx context.Context, readMessagePair <-chan *readWriteMessagePair) { +func (df *DataForward) concurrentApplyTransformer(ctx context.Context, readMessagePair <-chan *readWriteMessagePair) { for message := range readMessagePair { start := time.Now() - metrics.SourceTransformerReadMessagesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Inc() - writeMessages, err := isdf.applyTransformer(ctx, message.readMessage) - metrics.SourceTransformerWriteMessagesCount.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Add(float64(len(writeMessages))) + metrics.SourceTransformerReadMessagesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Inc() + writeMessages, err := df.applyTransformer(ctx, message.readMessage) + metrics.SourceTransformerWriteMessagesCount.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Add(float64(len(writeMessages))) message.writeMessages = append(message.writeMessages, writeMessages...) message.transformerError = err - metrics.SourceTransformerProcessingTime.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica)), metrics.LabelPartitionName: isdf.reader.GetName()}).Observe(float64(time.Since(start).Microseconds())) + metrics.SourceTransformerProcessingTime.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica)), metrics.LabelPartitionName: df.reader.GetName()}).Observe(float64(time.Since(start).Microseconds())) } } // applyTransformer applies the transformer and will block if there is any InternalErr. On the other hand, if this is a UserError // the skip flag is set. The ShutDown flag will only if there is an InternalErr and ForceStop has been invoked. // The UserError retry will be done on the applyTransformer. -func (isdf *DataForward) applyTransformer(ctx context.Context, readMessage *isb.ReadMessage) ([]*isb.WriteMessage, error) { +func (df *DataForward) applyTransformer(ctx context.Context, readMessage *isb.ReadMessage) ([]*isb.WriteMessage, error) { for { - writeMessages, err := isdf.transformer.ApplyTransform(ctx, readMessage) + writeMessages, err := df.transformer.ApplyTransform(ctx, readMessage) if err != nil { - isdf.opts.logger.Errorw("Transformer.Apply error", zap.Error(err)) + df.opts.logger.Errorw("Transformer.Apply error", zap.Error(err)) // TODO: implement retry with backoff etc. - time.Sleep(isdf.opts.retryInterval) + time.Sleep(df.opts.retryInterval) // keep retrying, I cannot think of a use case where a user could say, errors are fine :-) // as a platform, we should not lose or corrupt data. // this does not mean we should prohibit this from a shutdown. - if ok, _ := isdf.IsShuttingDown(); ok { - isdf.opts.logger.Errorw("Transformer.Apply, Stop called while stuck on an internal error", zap.Error(err)) - metrics.PlatformError.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica))}).Inc() + if ok, _ := df.IsShuttingDown(); ok { + df.opts.logger.Errorw("Transformer.Apply, Stop called while stuck on an internal error", zap.Error(err)) + metrics.PlatformError.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica))}).Inc() return nil, err } continue } else { for index, m := range writeMessages { - m.ID = fmt.Sprintf("%s-%s-%d", readMessage.ReadOffset.String(), isdf.vertexName, index) + m.ID = fmt.Sprintf("%s-%s-%d", readMessage.ReadOffset.String(), df.vertexName, index) } return writeMessages, nil } @@ -527,15 +570,15 @@ func (isdf *DataForward) applyTransformer(ctx context.Context, readMessage *isb. } // whereToStep executes the WhereTo interfaces and then updates the to step's writeToBuffers buffer. -func (isdf *DataForward) whereToStep(writeMessage *isb.WriteMessage, messageToStep map[string][][]isb.Message, readMessage *isb.ReadMessage) error { +func (df *DataForward) whereToStep(writeMessage *isb.WriteMessage, messageToStep map[string][][]isb.Message, readMessage *isb.ReadMessage) error { // call WhereTo and drop it on errors - to, err := isdf.toWhichStepDecider.WhereTo(writeMessage.Keys, writeMessage.Tags) + to, err := df.toWhichStepDecider.WhereTo(writeMessage.Keys, writeMessage.Tags) if err != nil { - isdf.opts.logger.Errorw("failed in whereToStep", zap.Error(isb.MessageWriteErr{Name: isdf.reader.GetName(), Header: readMessage.Header, Body: readMessage.Body, Message: fmt.Sprintf("WhereTo failed, %s", err)})) + df.opts.logger.Errorw("failed in whereToStep", zap.Error(isb.MessageWriteErr{Name: df.reader.GetName(), Header: readMessage.Header, Body: readMessage.Body, Message: fmt.Sprintf("WhereTo failed, %s", err)})) // a shutdown can break the blocking loop caused due to InternalErr - if ok, _ := isdf.IsShuttingDown(); ok { + if ok, _ := df.IsShuttingDown(); ok { err := fmt.Errorf("whereToStep, Stop called while stuck on an internal error, %v", err) - metrics.PlatformError.With(map[string]string{metrics.LabelVertex: isdf.vertexName, metrics.LabelPipeline: isdf.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(isdf.vertexReplica))}).Inc() + metrics.PlatformError.With(map[string]string{metrics.LabelVertex: df.vertexName, metrics.LabelPipeline: df.pipelineName, metrics.LabelVertexType: string(dfv1.VertexTypeSource), metrics.LabelVertexReplicaIndex: strconv.Itoa(int(df.vertexReplica))}).Inc() return err } return err @@ -543,7 +586,7 @@ func (isdf *DataForward) whereToStep(writeMessage *isb.WriteMessage, messageToSt for _, t := range to { if _, ok := messageToStep[t.ToVertexName]; !ok { - isdf.opts.logger.Errorw("failed in whereToStep", zap.Error(isb.MessageWriteErr{Name: isdf.reader.GetName(), Header: readMessage.Header, Body: readMessage.Body, Message: fmt.Sprintf("no such destination (%s)", t.ToVertexName)})) + df.opts.logger.Errorw("failed in whereToStep", zap.Error(isb.MessageWriteErr{Name: df.reader.GetName(), Header: readMessage.Header, Body: readMessage.Body, Message: fmt.Sprintf("no such destination (%s)", t.ToVertexName)})) } messageToStep[t.ToVertexName][t.ToVertexPartitionIdx] = append(messageToStep[t.ToVertexName][t.ToVertexPartitionIdx], writeMessage.Message) } @@ -551,15 +594,15 @@ func (isdf *DataForward) whereToStep(writeMessage *isb.WriteMessage, messageToSt } // createToVertexWatermarkPublisher creates a watermark publisher for the given toVertexName and partition -func (isdf *DataForward) createToVertexWatermarkPublisher(toVertexName string, partition int32) publish.Publisher { +func (df *DataForward) createToVertexWatermarkPublisher(toVertexName string, partition int32) publish.Publisher { - wmStore := isdf.toVertexWMStores[toVertexName] - entityName := fmt.Sprintf("%s-%s-%d", isdf.pipelineName, isdf.vertexName, partition) + wmStore := df.toVertexWMStores[toVertexName] + entityName := fmt.Sprintf("%s-%s-%d", df.pipelineName, df.vertexName, partition) processorEntity := entity.NewProcessorEntity(entityName) // if watermark is disabled, wmStore here is a no op store - publisher := publish.NewPublish(isdf.ctx, processorEntity, wmStore, int32(len(isdf.toBuffers[toVertexName]))) - isdf.toVertexWMPublishers[toVertexName][partition] = publisher + publisher := publish.NewPublish(df.ctx, processorEntity, wmStore, int32(len(df.toBuffers[toVertexName]))) + df.toVertexWMPublishers[toVertexName][partition] = publisher return publisher } diff --git a/pkg/sources/forward/data_forward_test.go b/pkg/sources/forward/data_forward_test.go index 41f103776e..9ab1519011 100644 --- a/pkg/sources/forward/data_forward_test.go +++ b/pkg/sources/forward/data_forward_test.go @@ -53,6 +53,38 @@ var ( testSourceWatermark = time.Unix(1636460000, 0).UTC() ) +type SimpleSource struct { + buffer *simplebuffer.InMemoryBuffer +} + +func NewSimpleSource(buffer *simplebuffer.InMemoryBuffer) *SimpleSource { + return &SimpleSource{buffer: buffer} +} + +func (s *SimpleSource) Close() error { + return s.buffer.Close() +} + +func (s *SimpleSource) GetName() string { + return s.buffer.GetName() +} + +func (s *SimpleSource) Read(ctx context.Context, i int64) ([]*isb.ReadMessage, error) { + return s.buffer.Read(ctx, i) +} + +func (s *SimpleSource) Ack(ctx context.Context, offsets []isb.Offset) []error { + return s.buffer.Ack(ctx, offsets) +} + +func (s *SimpleSource) Partitions() []int32 { + return []int32{0} +} + +func (s *SimpleSource) Write(ctx context.Context, messages []isb.Message) ([]isb.Offset, []error) { + return s.buffer.Write(ctx, messages) +} + type testForwardFetcher struct { // for data_forward_test.go only } @@ -63,7 +95,7 @@ func TestMain(m *testing.M) { // ComputeWatermark uses current time as the watermark because we want to make sure // the test publisher is publishing watermark -func (t *testForwardFetcher) ComputeWatermark(_ isb.Offset, _ int32) wmb.Watermark { +func (t *testForwardFetcher) ComputeWatermark() wmb.Watermark { return t.getWatermark() } @@ -71,9 +103,8 @@ func (t *testForwardFetcher) getWatermark() wmb.Watermark { return wmb.Watermark(testSourceWatermark) } -func (t *testForwardFetcher) ComputeHeadIdleWMB(int32) wmb.WMB { - // won't be used - return wmb.WMB{} +func (t *testForwardFetcher) ComputeHeadWatermark(int32) wmb.Watermark { + return wmb.Watermark{} } type myForwardTest struct { @@ -104,7 +135,7 @@ func TestNewDataForward(t *testing.T) { t.Run(tt.name+"_basic", func(t *testing.T) { metricsReset() batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to11 := simplebuffer.NewInMemoryBuffer("to1-1", 2*batchSize, 0) to12 := simplebuffer.NewInMemoryBuffer("to1-2", 2*batchSize, 1) toSteps := map[string][]isb.BufferWriter{ @@ -128,7 +159,7 @@ func TestNewDataForward(t *testing.T) { writeMessages := testutils.BuildTestWriteMessages(4*batchSize, testStartTime) - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toSteps) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toSteps) noOpStores := buildNoOpToVertexStores(toSteps) f, err := NewDataForward(vertexInstance, fromStep, toSteps, &mySourceForwardTestRoundRobin{}, myForwardTest{}, fetchWatermark, TestSourceWatermarkPublisher{}, noOpStores, wmb.NewIdleManager(len(toSteps)), WithReadBatchSize(batchSize)) @@ -200,7 +231,7 @@ func TestNewDataForward(t *testing.T) { // Explicitly tests the case where we forward to all buffers t.Run(tt.name+"_toAll", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 10*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 10*batchSize, 0)) to11 := simplebuffer.NewInMemoryBuffer("to1-1", 2*batchSize, 0) to12 := simplebuffer.NewInMemoryBuffer("to1-2", 2*batchSize, 1) @@ -351,7 +382,7 @@ func TestNewDataForward(t *testing.T) { // Explicitly tests the case where we drop all events t.Run(tt.name+"_dropAll", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to11 := simplebuffer.NewInMemoryBuffer("to1-1", 2*batchSize, 0) to12 := simplebuffer.NewInMemoryBuffer("to1-2", 2*batchSize, 1) @@ -516,7 +547,7 @@ func TestNewDataForward(t *testing.T) { // Explicitly tests the case where we forward to only one buffer t.Run(tt.name+"_toOneStep", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to11 := simplebuffer.NewInMemoryBuffer("to1-1", 2*batchSize, 0) to12 := simplebuffer.NewInMemoryBuffer("to1-2", 2*batchSize, 1) to21 := simplebuffer.NewInMemoryBuffer("to2-1", 2*batchSize, 0) @@ -655,7 +686,7 @@ func TestNewDataForward(t *testing.T) { // Test the scenario with Transformer error t.Run(tt.name+"_TransformerError", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to1 := simplebuffer.NewInMemoryBuffer("to1", 2*batchSize, 0) toSteps := map[string][]isb.BufferWriter{ "to1": {to1}, @@ -678,7 +709,7 @@ func TestNewDataForward(t *testing.T) { writeMessages := testutils.BuildTestWriteMessages(4*batchSize, testStartTime) - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toSteps) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toSteps) toVertexStores := buildNoOpToVertexStores(toSteps) f, err := NewDataForward(vertexInstance, fromStep, toSteps, myForwardApplyTransformerErrTest{}, myForwardApplyTransformerErrTest{}, fetchWatermark, TestSourceWatermarkPublisher{}, toVertexStores, wmb.NewIdleManager(len(toSteps)), WithReadBatchSize(batchSize)) @@ -700,7 +731,7 @@ func TestNewDataForward(t *testing.T) { // Test the scenario with error t.Run(tt.name+"_whereToError", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to1 := simplebuffer.NewInMemoryBuffer("to1", 2*batchSize, 0) toSteps := map[string][]isb.BufferWriter{ "to1": {to1}, @@ -722,7 +753,7 @@ func TestNewDataForward(t *testing.T) { Replica: 0, } - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toSteps) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toSteps) toVertexStores := buildNoOpToVertexStores(toSteps) f, err := NewDataForward(vertexInstance, fromStep, toSteps, myForwardApplyWhereToErrTest{}, myForwardApplyWhereToErrTest{}, fetchWatermark, TestSourceWatermarkPublisher{}, toVertexStores, wmb.NewIdleManager(len(toSteps)), WithReadBatchSize(batchSize)) @@ -742,7 +773,7 @@ func TestNewDataForward(t *testing.T) { }) t.Run(tt.name+"_withInternalError", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to1 := simplebuffer.NewInMemoryBuffer("to1", 2*batchSize, 0) toSteps := map[string][]isb.BufferWriter{ "to1": {to1}, @@ -764,7 +795,7 @@ func TestNewDataForward(t *testing.T) { Replica: 0, } - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toSteps) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toSteps) toVertexStores := buildNoOpToVertexStores(toSteps) f, err := NewDataForward(vertexInstance, fromStep, toSteps, myForwardInternalErrTest{}, myForwardInternalErrTest{}, fetchWatermark, TestSourceWatermarkPublisher{}, toVertexStores, wmb.NewIdleManager(len(toSteps)), WithReadBatchSize(batchSize)) @@ -845,12 +876,16 @@ func (f mySourceForwardTest) ApplyTransform(ctx context.Context, message *isb.Re type TestSourceWatermarkPublisher struct { } +func (p TestSourceWatermarkPublisher) PublishIdleWatermarks(time.Time, []int32) { + // PublishIdleWatermarks is not tested in data_forwarder_test.go +} + func (p TestSourceWatermarkPublisher) PublishSourceWatermarks([]*isb.ReadMessage) { // PublishSourceWatermarks is not tested in data_forwarder_test.go } func TestDataForwardSinglePartition(t *testing.T) { - fromStep := simplebuffer.NewInMemoryBuffer("from", 25, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 25, 0)) to1 := simplebuffer.NewInMemoryBuffer("to1", 10, 0, simplebuffer.WithReadTimeOut(time.Second*10)) toSteps := map[string][]isb.BufferWriter{ "to1": {to1}, @@ -903,7 +938,7 @@ func TestDataForwardSinglePartition(t *testing.T) { } func TestDataForwardMultiplePartition(t *testing.T) { - fromStep := simplebuffer.NewInMemoryBuffer("from", 25, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 25, 0)) to11 := simplebuffer.NewInMemoryBuffer("to1-0", 10, 0, simplebuffer.WithReadTimeOut(time.Second*10)) to12 := simplebuffer.NewInMemoryBuffer("to1-1", 10, 1, simplebuffer.WithReadTimeOut(time.Second*10)) toSteps := map[string][]isb.BufferWriter{ @@ -1013,7 +1048,7 @@ func TestWriteToBuffer(t *testing.T) { } for _, value := range tests { t.Run(value.name, func(t *testing.T) { - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*value.batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*value.batchSize, 0)) buffer := simplebuffer.NewInMemoryBuffer("to1", value.batchSize, 0, simplebuffer.WithBufferFullWritingStrategy(value.strategy)) toSteps := map[string][]isb.BufferWriter{ "to1": {buffer}, @@ -1031,7 +1066,7 @@ func TestWriteToBuffer(t *testing.T) { Replica: 0, } - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toSteps) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toSteps) toVertexStores := buildNoOpToVertexStores(toSteps) f, err := NewDataForward(vertexInstance, fromStep, toSteps, myForwardTest{}, myForwardTest{}, fetchWatermark, TestSourceWatermarkPublisher{}, toVertexStores, wmb.NewIdleManager(len(toSteps)), WithReadBatchSize(value.batchSize)) assert.NoError(t, err) diff --git a/pkg/sources/forward/shutdown.go b/pkg/sources/forward/shutdown.go index 3bf61c7d89..e195f044df 100644 --- a/pkg/sources/forward/shutdown.go +++ b/pkg/sources/forward/shutdown.go @@ -32,11 +32,11 @@ type Shutdown struct { } // IsShuttingDown returns whether we can stop processing. -func (isdf *DataForward) IsShuttingDown() (bool, error) { - isdf.Shutdown.rwlock.RLock() - defer isdf.Shutdown.rwlock.RUnlock() +func (df *DataForward) IsShuttingDown() (bool, error) { + df.Shutdown.rwlock.RLock() + defer df.Shutdown.rwlock.RUnlock() - if isdf.Shutdown.forceShutdown || isdf.Shutdown.startShutdown { + if df.Shutdown.forceShutdown || df.Shutdown.startShutdown { return true, nil } @@ -51,24 +51,24 @@ func (s *Shutdown) String() string { } // Stop stops the processing. -func (isdf *DataForward) Stop() { - isdf.Shutdown.rwlock.Lock() - defer isdf.Shutdown.rwlock.Unlock() - if isdf.Shutdown.initiateTime.IsZero() { - isdf.Shutdown.initiateTime = time.Now() +func (df *DataForward) Stop() { + df.Shutdown.rwlock.Lock() + defer df.Shutdown.rwlock.Unlock() + if df.Shutdown.initiateTime.IsZero() { + df.Shutdown.initiateTime = time.Now() } - isdf.Shutdown.startShutdown = true - isdf.Shutdown.shutdownRequestCtr++ + df.Shutdown.startShutdown = true + df.Shutdown.shutdownRequestCtr++ // call cancel - isdf.cancelFn() + df.cancelFn() } // ForceStop sets up the force shutdown flag. -func (isdf *DataForward) ForceStop() { +func (df *DataForward) ForceStop() { // call stop (what if we have an enthusiastic shutdown that forces first) // e.g., I know I have written a wrong source transformer, so shutdown ASAP - isdf.Stop() - isdf.Shutdown.rwlock.Lock() - defer isdf.Shutdown.rwlock.Unlock() - isdf.Shutdown.forceShutdown = true + df.Stop() + df.Shutdown.rwlock.Lock() + defer df.Shutdown.rwlock.Unlock() + df.Shutdown.forceShutdown = true } diff --git a/pkg/sources/forward/shutdown_test.go b/pkg/sources/forward/shutdown_test.go index 49b7e53558..21ed2f89b6 100644 --- a/pkg/sources/forward/shutdown_test.go +++ b/pkg/sources/forward/shutdown_test.go @@ -64,7 +64,7 @@ func TestInterStepDataForward(t *testing.T) { for _, tt := range tests { t.Run(tt.name+"_stop", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to1 := simplebuffer.NewInMemoryBuffer("to1", 2*batchSize, 0) toSteps := map[string][]isb.BufferWriter{ "to1": {to1}, @@ -86,7 +86,7 @@ func TestInterStepDataForward(t *testing.T) { Replica: 0, } - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toSteps) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toSteps) toVertexWmStores := buildNoOpToVertexStores(toSteps) f, err := NewDataForward(vertexInstance, fromStep, toSteps, myShutdownTest{}, myShutdownTest{}, fetchWatermark, TestSourceWatermarkPublisher{}, toVertexWmStores, wmb.NewIdleManager(len(toSteps)), WithReadBatchSize(batchSize)) assert.NoError(t, err) @@ -102,7 +102,7 @@ func TestInterStepDataForward(t *testing.T) { }) t.Run(tt.name+"_forceStop", func(t *testing.T) { batchSize := tt.batchSize - fromStep := simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0) + fromStep := NewSimpleSource(simplebuffer.NewInMemoryBuffer("from", 5*batchSize, 0)) to1 := simplebuffer.NewInMemoryBuffer("to", 2*batchSize, 0) toSteps := map[string][]isb.BufferWriter{ "to1": {to1}, @@ -124,7 +124,7 @@ func TestInterStepDataForward(t *testing.T) { Replica: 0, } - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toSteps) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toSteps) toVertexWmStores := buildNoOpToVertexStores(toSteps) f, err := NewDataForward(vertexInstance, fromStep, toSteps, myShutdownTest{}, myShutdownTest{}, fetchWatermark, TestSourceWatermarkPublisher{}, toVertexWmStores, wmb.NewIdleManager(len(toSteps)), WithReadBatchSize(batchSize)) assert.NoError(t, err) diff --git a/pkg/sources/generator/tickgen.go b/pkg/sources/generator/tickgen.go index c637b7c6fa..930ae15d86 100644 --- a/pkg/sources/generator/tickgen.go +++ b/pkg/sources/generator/tickgen.go @@ -34,7 +34,7 @@ import ( "github.com/numaproj/numaflow/pkg/shared/logging" sourceforward "github.com/numaproj/numaflow/pkg/sources/forward" applier2 "github.com/numaproj/numaflow/pkg/sources/forward/applier" - "github.com/numaproj/numaflow/pkg/watermark/entity" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/watermark/fetch" "github.com/numaproj/numaflow/pkg/watermark/publish" "github.com/numaproj/numaflow/pkg/watermark/store" @@ -92,56 +92,36 @@ var recordGenerator = func(size int32, value *uint64, createdTS int64) []byte { return marshalled } -type memgen struct { - // srcChan provides a go channel that supplies generated data - srcChan chan record - // rpu - records per time unit - rpu int - // keyCount is the number of unique keys in the payload - keyCount int32 - // value is the optional uint64 number that can be set in the payload - // can be retrieved in the udf - value *uint64 - // msgSize is the size of each generated message - msgSize int32 - // timeunit - ticker will fire once per timeunit and generates - // a number of records equal to the number passed to rpu. - timeunit time.Duration - // genFn function that generates a payload as a byte array - genFn func(int32, *uint64, int64) []byte - // name is the name of the source vertex - vertexName string - // pipelineName is the name of the pipeline - pipelineName string - // cancelFn terminates the source will not generate any more records. - cancelFn context.CancelFunc - // forwarder to read from the source and write to the inter step buffer. - forwarder *sourceforward.DataForward - // lifecycleCtx context is used to control the lifecycle of this instance. - lifecycleCtx context.Context - // read timeout for the reader - readTimeout time.Duration - - // vertex instance - vertexInstance *dfv1.VertexInstance - // source watermark publisher - sourcePublishWM publish.Publisher - - logger *zap.SugaredLogger +type memGen struct { + srcChan chan record // srcChan provides a go channel that supplies generated data + rpu int // rpu - records per time unit + keyCount int32 // keyCount is the number of unique keys in the payload + value *uint64 // value is the optional uint64 number that can be set in the payload + msgSize int32 // msgSize is the size of each generated message + timeunit time.Duration // timeunit - ticker will fire once per timeunit + genFn func(int32, *uint64, int64) []byte // genFn function that generates a payload as a byte array + vertexName string // name is the name of the source vertex + pipelineName string // pipelineName is the name of the pipeline + cancelFn context.CancelFunc // cancelFn terminates the source will not generate any more records. + forwarder *sourceforward.DataForward // forwarder to read from the source and write to the inter step buffer. + lifecycleCtx context.Context // lifecycleCtx context is used to control the lifecycle of this instance. + readTimeout time.Duration // read timeout for the reader + vertexInstance *dfv1.VertexInstance // vertex instance + logger *zap.SugaredLogger } -type Option func(*memgen) error +type Option func(*memGen) error // WithLogger is used to return logger information func WithLogger(l *zap.SugaredLogger) Option { - return func(o *memgen) error { + return func(o *memGen) error { o.logger = l return nil } } func WithReadTimeout(timeout time.Duration) Option { - return func(o *memgen) error { + return func(o *memGen) error { o.readTimeout = timeout return nil } @@ -153,11 +133,11 @@ func NewMemGen( writers map[string][]isb.BufferWriter, fsd forwarder.ToWhichStepDecider, transformerApplier applier2.SourceTransformApplier, - fetchWM fetch.Fetcher, + fetchWM fetch.SourceFetcher, toVertexPublisherStores map[string]store.WatermarkStore, publishWMStores store.WatermarkStore, idleManager wmb.IdleManager, - opts ...Option) (*memgen, error) { + opts ...Option) (sourcer.Sourcer, error) { // minimal CRDs don't have defaults rpu := 5 @@ -181,7 +161,7 @@ func NewMemGen( value = vertexInstance.Vertex.Spec.Source.Generator.Value } - genSrc := &memgen{ + genSrc := &memGen{ rpu: rpu, keyCount: keyCount, value: value, @@ -205,9 +185,9 @@ func NewMemGen( } // this context is to be used internally for controlling the lifecycle of generator - cctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) - genSrc.lifecycleCtx = cctx + genSrc.lifecycleCtx = ctx genSrc.cancelFn = cancel forwardOpts := []sourceforward.Option{sourceforward.WithLogger(genSrc.logger)} @@ -217,42 +197,35 @@ func NewMemGen( } } - // attach a source publisher so the source can assign the watermarks. - genSrc.sourcePublishWM = genSrc.buildSourceWatermarkPublisher(publishWMStores) + // create a source watermark publisher + sourceWmPublisher := publish.NewSourcePublish(ctx, genSrc.pipelineName, genSrc.vertexName, publishWMStores, + publish.WithDelay(vertexInstance.Vertex.Spec.Watermark.GetMaxDelay()), publish.WithDefaultPartitionIdx(vertexInstance.Replica)) - // we pass in the context to forwarder as well so that it can shut down when we cancelFn the context - forwarder, err := sourceforward.NewDataForward(vertexInstance, genSrc, writers, fsd, transformerApplier, fetchWM, genSrc, toVertexPublisherStores, idleManager, forwardOpts...) + // we pass in the context to sourceForwarder as well so that it can shut down when we cancelFn the context + sourceForwarder, err := sourceforward.NewDataForward(vertexInstance, genSrc, writers, fsd, transformerApplier, fetchWM, sourceWmPublisher, toVertexPublisherStores, idleManager, forwardOpts...) if err != nil { return nil, err } - genSrc.forwarder = forwarder + genSrc.forwarder = sourceForwarder return genSrc, nil } -func (mg *memgen) buildSourceWatermarkPublisher(publishWMStores store.WatermarkStore) publish.Publisher { - // for tickgen, it can be the name of the replica - entityName := fmt.Sprintf("%s-%d", mg.vertexInstance.Vertex.Name, mg.vertexInstance.Replica) - processorEntity := entity.NewProcessorEntity(entityName) - // source publisher toVertexPartitionCount will be 1, because we publish watermarks within the source itself. - return publish.NewPublish(mg.lifecycleCtx, processorEntity, publishWMStores, 1, publish.IsSource(), publish.WithDelay(mg.vertexInstance.Vertex.Spec.Watermark.GetMaxDelay())) -} - -func (mg *memgen) GetName() string { +// GetName returns the name of the source +func (mg *memGen) GetName() string { return mg.vertexName } -// GetPartitionIdx returns the partition number for the source vertex buffer -// Source is like a buffer with only one partition. So, we always return 0 -func (mg *memgen) GetPartitionIdx() int32 { - return 0 +// Partitions returns the partitions for the source. +func (mg *memGen) Partitions() []int32 { + return []int32{mg.vertexInstance.Replica} } -func (mg *memgen) IsEmpty() bool { +func (mg *memGen) IsEmpty() bool { return len(mg.srcChan) == 0 } -func (mg *memgen) Read(_ context.Context, count int64) ([]*isb.ReadMessage, error) { +func (mg *memGen) Read(_ context.Context, count int64) ([]*isb.ReadMessage, error) { msgs := make([]*isb.ReadMessage, 0, count) // timeout should not be re-triggered for every run of the for loop. it is for the entire Read() call. timeout := time.After(mg.readTimeout) @@ -272,36 +245,25 @@ loop: return msgs, nil } -func (mg *memgen) Pending(_ context.Context) (int64, error) { +func (mg *memGen) Pending(_ context.Context) (int64, error) { return isb.PendingNotAvailable, nil } -func (mg *memgen) PublishSourceWatermarks(msgs []*isb.ReadMessage) { - if len(msgs) <= 0 { - return - } - // use the first event time of the message as watermark to make it conservative - // toVertexPartitionCount is 1 because we publish watermarks within source itself. - mg.sourcePublishWM.PublishWatermark(wmb.Watermark(msgs[0].EventTime), nil, 0) // Source publisher does not care about the offset -} - // Ack acknowledges an array of offset. -func (mg *memgen) Ack(_ context.Context, offsets []isb.Offset) []error { +func (mg *memGen) Ack(_ context.Context, offsets []isb.Offset) []error { return make([]error, len(offsets)) } -func (mg *memgen) NoAck(_ context.Context, _ []isb.Offset) {} - -func (mg *memgen) Close() error { +func (mg *memGen) Close() error { return nil } -func (mg *memgen) Stop() { +func (mg *memGen) Stop() { mg.cancelFn() mg.forwarder.Stop() } -func (mg *memgen) ForceStop() { +func (mg *memGen) ForceStop() { mg.Stop() mg.forwarder.ForceStop() @@ -310,12 +272,12 @@ func (mg *memgen) ForceStop() { // Start starts reading from the source // context is used to control the lifecycle of this component. // this context will be used to shut down the vertex once an os.signal is received. -func (mg *memgen) Start() <-chan struct{} { +func (mg *memGen) Start() <-chan struct{} { mg.generator(mg.lifecycleCtx, mg.rpu, mg.timeunit) return mg.forwarder.Start() } -func (mg *memgen) NewWorker(ctx context.Context, rate int) func(chan time.Time, chan struct{}) { +func (mg *memGen) NewWorker(ctx context.Context, rate int) func(chan time.Time, chan struct{}) { return func(tickChan chan time.Time, done chan struct{}) { defer func() { @@ -359,7 +321,7 @@ func (mg *memgen) NewWorker(ctx context.Context, rate int) func(chan time.Time, } // generator fires once per time unit and generates records and writes them to the channel -func (mg *memgen) generator(ctx context.Context, rate int, timeunit time.Duration) { +func (mg *memGen) generator(ctx context.Context, rate int, timeunit time.Duration) { go func() { // capping the rate to 10000 msgs/sec if rate > 10000 { @@ -397,7 +359,7 @@ func (mg *memgen) generator(ctx context.Context, rate int, timeunit time.Duratio }() } -func (mg *memgen) newReadMessage(key string, payload []byte, offset int64) *isb.ReadMessage { +func (mg *memGen) newReadMessage(key string, payload []byte, offset int64) *isb.ReadMessage { readOffset := isb.NewSimpleIntPartitionOffset(offset, mg.vertexInstance.Replica) msg := isb.Message{ Header: isb.Header{ diff --git a/pkg/sources/generator/tickgen_test.go b/pkg/sources/generator/tickgen_test.go index 5127c43a67..4a50b4ca13 100644 --- a/pkg/sources/generator/tickgen_test.go +++ b/pkg/sources/generator/tickgen_test.go @@ -49,7 +49,7 @@ func TestRead(t *testing.T) { ctx := context.Background() vertex := &dfv1.Vertex{ ObjectMeta: v1.ObjectMeta{ - Name: "memgen", + Name: "memGen", }, Spec: dfv1.VertexSpec{ PipelineName: "testPipeline", @@ -72,7 +72,7 @@ func TestRead(t *testing.T) { "writer": {dest}, } - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toBuffers) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toBuffers) toVertexWmStores := map[string]store.WatermarkStore{ "writer": publishWMStore, } @@ -89,7 +89,7 @@ func TestRead(t *testing.T) { // wait for the context to be completely stopped. for { - _, ok := <-mgen.srcChan + _, ok := <-mgen.(*memGen).srcChan if !ok { break } @@ -109,7 +109,7 @@ func TestStop(t *testing.T) { dest := simplebuffer.NewInMemoryBuffer("writer", 10, 0) vertex := &dfv1.Vertex{ ObjectMeta: v1.ObjectMeta{ - Name: "memgen", + Name: "memGen", }, Spec: dfv1.VertexSpec{ PipelineName: "testPipeline", @@ -131,7 +131,7 @@ func TestStop(t *testing.T) { "writer": {dest}, } - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(toBuffers) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(toBuffers) toVertexWmStores := map[string]store.WatermarkStore{ "writer": publishWMStore, } @@ -209,7 +209,7 @@ func TestWatermark(t *testing.T) { dest := simplebuffer.NewInMemoryBuffer("writer", 1000, 0) vertex := &dfv1.Vertex{ ObjectMeta: v1.ObjectMeta{ - Name: "memgen", + Name: "memGen", }, Spec: dfv1.VertexSpec{ PipelineName: "testPipeline", diff --git a/pkg/sources/http/http.go b/pkg/sources/http/http.go index 388c1a6366..e1b8b8b102 100644 --- a/pkg/sources/http/http.go +++ b/pkg/sources/http/http.go @@ -38,7 +38,7 @@ import ( sharedutil "github.com/numaproj/numaflow/pkg/shared/util" sourceforward "github.com/numaproj/numaflow/pkg/sources/forward" "github.com/numaproj/numaflow/pkg/sources/forward/applier" - "github.com/numaproj/numaflow/pkg/watermark/entity" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/watermark/fetch" "github.com/numaproj/numaflow/pkg/watermark/publish" "github.com/numaproj/numaflow/pkg/watermark/store" @@ -46,19 +46,17 @@ import ( ) type httpSource struct { - vertexName string - pipelineName string - ready bool - readTimeout time.Duration - bufferSize int - messages chan *isb.ReadMessage - logger *zap.SugaredLogger - forwarder *sourceforward.DataForward - // source watermark publisher - sourcePublishWM publish.Publisher - // context cancel function - cancelFunc context.CancelFunc - shutdown func(context.Context) error + vertexName string + pipelineName string + vertexReplica int32 + ready bool + readTimeout time.Duration + bufferSize int + messages chan *isb.ReadMessage + logger *zap.SugaredLogger + forwarder *sourceforward.DataForward + cancelFunc context.CancelFunc // context cancel function + shutdown func(context.Context) error } type Option func(*httpSource) error @@ -91,18 +89,19 @@ func New( writers map[string][]isb.BufferWriter, fsd forwarder.ToWhichStepDecider, transformerApplier applier.SourceTransformApplier, - fetchWM fetch.Fetcher, + fetchWM fetch.SourceFetcher, toVertexPublisherStores map[string]store.WatermarkStore, publishWMStores store.WatermarkStore, idleManager wmb.IdleManager, - opts ...Option) (*httpSource, error) { + opts ...Option) (sourcer.Sourcer, error) { h := &httpSource{ - vertexName: vertexInstance.Vertex.Spec.Name, - pipelineName: vertexInstance.Vertex.Spec.PipelineName, - ready: false, - bufferSize: 1000, // default size - readTimeout: 1 * time.Second, // default timeout + vertexName: vertexInstance.Vertex.Spec.Name, + pipelineName: vertexInstance.Vertex.Spec.PipelineName, + vertexReplica: vertexInstance.Replica, + ready: false, + bufferSize: 1000, // default size + readTimeout: 1 * time.Second, // default timeout } for _, o := range opts { @@ -199,30 +198,29 @@ func New( forwardOpts = append(forwardOpts, sourceforward.WithReadBatchSize(int64(*x.ReadBatchSize))) } } + ctx, cancel := context.WithCancel(context.Background()) + h.cancelFunc = cancel - h.forwarder, err = sourceforward.NewDataForward(vertexInstance, h, writers, fsd, transformerApplier, fetchWM, h, toVertexPublisherStores, idleManager, forwardOpts...) + // create a source watermark publisher + sourceWmPublisher := publish.NewSourcePublish(ctx, vertexInstance.Vertex.Spec.PipelineName, vertexInstance.Vertex.Spec.Name, + publishWMStores, publish.WithDelay(vertexInstance.Vertex.Spec.Watermark.GetMaxDelay()), publish.WithDefaultPartitionIdx(vertexInstance.Replica)) + h.forwarder, err = sourceforward.NewDataForward(vertexInstance, h, writers, fsd, transformerApplier, fetchWM, sourceWmPublisher, toVertexPublisherStores, idleManager, forwardOpts...) if err != nil { h.logger.Errorw("Error instantiating the forwarder", zap.Error(err)) return nil, err } - ctx, cancel := context.WithCancel(context.Background()) - h.cancelFunc = cancel - entityName := fmt.Sprintf("%s-%d", vertexInstance.Vertex.Name, vertexInstance.Replica) - processorEntity := entity.NewProcessorEntity(entityName) - // source publisher toVertexPartitionCount will be 1, because we publish watermarks within the source itself. - h.sourcePublishWM = publish.NewPublish(ctx, processorEntity, publishWMStores, 1, publish.IsSource(), publish.WithDelay(vertexInstance.Vertex.Spec.Watermark.GetMaxDelay())) return h, nil } +// GetName returns the name of the source. func (h *httpSource) GetName() string { return h.vertexName } -// GetPartitionIdx returns the partition number for the source vertex buffer -// Source is like a buffer with only one partition. So, we always return 0 -func (h *httpSource) GetPartitionIdx() int32 { - return 0 +// Partitions returns the partitions for the source. +func (h *httpSource) Partitions() []int32 { + return []int32{h.vertexReplica} } func (h *httpSource) Read(_ context.Context, count int64) ([]*isb.ReadMessage, error) { @@ -247,26 +245,10 @@ func (h *httpSource) Pending(_ context.Context) (int64, error) { return isb.PendingNotAvailable, nil } -func (h *httpSource) PublishSourceWatermarks(msgs []*isb.ReadMessage) { - var oldest time.Time - for _, m := range msgs { - if oldest.IsZero() || m.EventTime.Before(oldest) { - oldest = m.EventTime - } - } - if len(msgs) > 0 && !oldest.IsZero() { - h.logger.Debugf("Publishing watermark %v to source", oldest) - // toVertexPartitionIdx is 0, because we publish watermarks within the source itself. - h.sourcePublishWM.PublishWatermark(wmb.Watermark(oldest), nil, 0) // Source publisher does not care about the offset - } -} - func (h *httpSource) Ack(_ context.Context, offsets []isb.Offset) []error { return make([]error, len(offsets)) } -func (h *httpSource) NoAck(_ context.Context, _ []isb.Offset) {} - func (h *httpSource) Close() error { h.logger.Info("Shutting down http source server...") h.cancelFunc() diff --git a/pkg/sources/http/http_test.go b/pkg/sources/http/http_test.go index 38133f896d..2a28bd42eb 100644 --- a/pkg/sources/http/http_test.go +++ b/pkg/sources/http/http_test.go @@ -81,18 +81,18 @@ func Test_NewHTTP(t *testing.T) { "test": {dest}, } publishWMStores, _ := store.BuildNoOpWatermarkStore() - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) toVertexWmStores := map[string]store.WatermarkStore{ "test": publishWMStores, } h, err := New(vi, toBuffers, myForwardToAllTest{}, applier.Terminal, fetchWatermark, toVertexWmStores, publishWMStores, wmb.NewIdleManager(len(toBuffers))) assert.NoError(t, err) - assert.False(t, h.ready) + assert.False(t, h.(*httpSource).ready) assert.Equal(t, v.Spec.Name, h.GetName()) - assert.NotNil(t, h.forwarder) - assert.NotNil(t, h.shutdown) + assert.NotNil(t, h.(*httpSource).forwarder) + assert.NotNil(t, h.(*httpSource).shutdown) _ = h.Start() - assert.True(t, h.ready) + assert.True(t, h.(*httpSource).ready) h.Stop() - assert.False(t, h.ready) + assert.False(t, h.(*httpSource).ready) } diff --git a/pkg/sources/kafka/handler.go b/pkg/sources/kafka/handler.go index ccd40813e8..4d65f7c3d5 100644 --- a/pkg/sources/kafka/handler.go +++ b/pkg/sources/kafka/handler.go @@ -27,9 +27,9 @@ import ( // consumerHandler struct type consumerHandler struct { - inflightacks chan bool + inflightAcks chan bool ready chan bool - readycloser sync.Once + readyCloser sync.Once messages chan *sarama.ConsumerMessage sess sarama.ConsumerGroupSession logger *zap.SugaredLogger @@ -47,7 +47,7 @@ func newConsumerHandler(readChanSize int) *consumerHandler { // Setup is run at the beginning of a new session, before ConsumeClaim func (consumer *consumerHandler) Setup(sess sarama.ConsumerGroupSession) error { consumer.sess = sess - consumer.readycloser.Do(func() { + consumer.readyCloser.Do(func() { close(consumer.ready) }) return nil @@ -56,7 +56,7 @@ func (consumer *consumerHandler) Setup(sess sarama.ConsumerGroupSession) error { // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited func (consumer *consumerHandler) Cleanup(sess sarama.ConsumerGroupSession) error { // wait for inflight acks to be completed. - <-consumer.inflightacks + <-consumer.inflightAcks sess.Commit() return nil } diff --git a/pkg/sources/kafka/handler_test.go b/pkg/sources/kafka/handler_test.go index 270ca331ec..1e9772e0aa 100644 --- a/pkg/sources/kafka/handler_test.go +++ b/pkg/sources/kafka/handler_test.go @@ -77,7 +77,7 @@ func TestMessageHandling(t *testing.T) { Replica: 0, } publishWMStore, _ := store.BuildNoOpWatermarkStore() - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) toVertexWmStores := map[string]store.WatermarkStore{ "test": publishWMStore, } @@ -94,7 +94,7 @@ func TestMessageHandling(t *testing.T) { expectedoffset := fmt.Sprintf("%s:%v:%v", topic, partition, offset) // push one message - ks.handler.messages <- msg + ks.(*kafkaSource).handler.messages <- msg readmsgs, err := ks.Read(context.Background(), 10) assert.Nil(t, err) diff --git a/pkg/sources/kafka/reader.go b/pkg/sources/kafka/reader.go index 8669aa7d16..f3de1ae6d4 100644 --- a/pkg/sources/kafka/reader.go +++ b/pkg/sources/kafka/reader.go @@ -33,53 +33,30 @@ import ( sharedutil "github.com/numaproj/numaflow/pkg/shared/util" sourceforward "github.com/numaproj/numaflow/pkg/sources/forward" "github.com/numaproj/numaflow/pkg/sources/forward/applier" - "github.com/numaproj/numaflow/pkg/watermark/entity" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/watermark/fetch" "github.com/numaproj/numaflow/pkg/watermark/publish" "github.com/numaproj/numaflow/pkg/watermark/store" "github.com/numaproj/numaflow/pkg/watermark/wmb" ) -type KafkaSource struct { - // name of the source vertex - vertexName string - // name of the pipeline - pipelineName string - // group name for the source vertex - groupName string - // topic to consume messages from - topic string - // kafka brokers - brokers []string - // forwarder that writes the consumed data to destination - forwarder *sourceforward.DataForward - // context cancel function - cancelFn context.CancelFunc - // lifecycle context - lifecycleCtx context.Context - // handler for a kafka consumer group - handler *consumerHandler - // sarama config for kafka consumer group - config *sarama.Config - // logger - logger *zap.SugaredLogger - // channel to indicate that we are done - stopCh chan struct{} - // size of the buffer that holds consumed but yet to be forwarded messages - handlerBuffer int - // read timeout for the from buffer - readTimeout time.Duration - // client used to calculate pending messages - adminClient sarama.ClusterAdmin - // sarama client - saramaClient sarama.Client - // source watermark publishers for different partitions - sourcePublishWMs map[int32]publish.Publisher - // max delay duration of watermark - watermarkMaxDelay time.Duration - // source watermark publisher stores - srcPublishWMStores store.WatermarkStore - lock *sync.RWMutex +type kafkaSource struct { + vertexName string // name of the source vertex + pipelineName string // name of the pipeline + groupName string // group name for the source vertex + topic string // topic to consume messages from + brokers []string // kafka brokers + forwarder *sourceforward.DataForward // forwarder that writes the consumed data to destination + cancelFn context.CancelFunc // context cancel function + lifecycleCtx context.Context // lifecycle context + handler *consumerHandler // handler for a kafka consumer group + config *sarama.Config // sarama config for kafka consumer group + logger *zap.SugaredLogger // logger + stopCh chan struct{} // channel to indicate that we are done + handlerBuffer int // size of the buffer that holds consumed but yet to be forwarded messages + readTimeout time.Duration // read timeout for the from buffer + adminClient sarama.ClusterAdmin // client used to calculate pending messages + saramaClient sarama.Client // sarama client } // kafkaOffset implements isb.Offset @@ -116,11 +93,11 @@ func (k *kafkaOffset) Topic() string { return k.topic } -type Option func(*KafkaSource) error +type Option func(*kafkaSource) error // WithLogger is used to return logger information func WithLogger(l *zap.SugaredLogger) Option { - return func(o *KafkaSource) error { + return func(o *kafkaSource) error { o.logger = l return nil } @@ -128,7 +105,7 @@ func WithLogger(l *zap.SugaredLogger) Option { // WithBufferSize is used to return size of message channel information func WithBufferSize(s int) Option { - return func(o *KafkaSource) error { + return func(o *kafkaSource) error { o.handlerBuffer = s return nil } @@ -136,7 +113,7 @@ func WithBufferSize(s int) Option { // WithReadTimeOut is used to set the read timeout for the from buffer func WithReadTimeOut(t time.Duration) Option { - return func(o *KafkaSource) error { + return func(o *kafkaSource) error { o.readTimeout = t return nil } @@ -144,23 +121,27 @@ func WithReadTimeOut(t time.Duration) Option { // WithGroupName is used to set the group name func WithGroupName(gn string) Option { - return func(o *KafkaSource) error { + return func(o *kafkaSource) error { o.groupName = gn return nil } } -func (r *KafkaSource) GetName() string { +func (r *kafkaSource) GetName() string { return r.vertexName } -// GetPartitionIdx returns the partition number for the source vertex buffer -// Source is like a buffer with only one partition. So, we always return 0 -func (r *KafkaSource) GetPartitionIdx() int32 { - return 0 +// Partitions returns the partitions from which the source is reading. +func (r *kafkaSource) Partitions() []int32 { + for topic, partitions := range r.handler.sess.Claims() { + if topic == r.topic { + return partitions + } + } + return []int32{} } -func (r *KafkaSource) Read(_ context.Context, count int64) ([]*isb.ReadMessage, error) { +func (r *kafkaSource) Read(_ context.Context, count int64) ([]*isb.ReadMessage, error) { msgs := make([]*isb.ReadMessage, 0, count) timeout := time.After(r.readTimeout) loop: @@ -178,42 +159,11 @@ loop: return msgs, nil } -func (r *KafkaSource) PublishSourceWatermarks(msgs []*isb.ReadMessage) { - // oldestTimestamps stores the latest timestamps for different partitions - oldestTimestamps := make(map[int32]time.Time) - for _, m := range msgs { - // Get latest timestamps for different partitions - if t, ok := oldestTimestamps[m.ReadOffset.PartitionIdx()]; !ok || m.EventTime.Before(t) { - oldestTimestamps[m.ReadOffset.PartitionIdx()] = m.EventTime - } - } - for p, t := range oldestTimestamps { - publisher := r.loadSourceWatermarkPublisher(p) - // toVertexPartitionIdx is 0 because we publish watermarks within the source itself. - publisher.PublishWatermark(wmb.Watermark(t), nil, 0) // Source publisher does not care about the offset - } -} - -// loadSourceWatermarkPublisher does a lazy load on the watermark publisher -func (r *KafkaSource) loadSourceWatermarkPublisher(partitionID int32) publish.Publisher { - r.lock.Lock() - defer r.lock.Unlock() - if p, ok := r.sourcePublishWMs[partitionID]; ok { - return p - } - entityName := fmt.Sprintf("%s-%s-%d", r.pipelineName, r.vertexName, partitionID) - processorEntity := entity.NewProcessorEntity(entityName) - // toVertexPartitionCount is 1 because we publish watermarks within the source itself. - sourcePublishWM := publish.NewPublish(r.lifecycleCtx, processorEntity, r.srcPublishWMStores, 1, publish.IsSource(), publish.WithDelay(r.watermarkMaxDelay)) - r.sourcePublishWMs[partitionID] = sourcePublishWM - return sourcePublishWM -} - // Ack acknowledges an array of offset. -func (r *KafkaSource) Ack(_ context.Context, offsets []isb.Offset) []error { +func (r *kafkaSource) Ack(_ context.Context, offsets []isb.Offset) []error { // we want to block the handler from exiting if there are any inflight acks. - r.handler.inflightacks = make(chan bool) - defer close(r.handler.inflightacks) + r.handler.inflightAcks = make(chan bool) + defer close(r.handler.inflightAcks) for _, offset := range offsets { topic := offset.(*kafkaOffset).Topic() @@ -233,9 +183,7 @@ func (r *KafkaSource) Ack(_ context.Context, offsets []isb.Offset) []error { return make([]error, len(offsets)) } -func (r *KafkaSource) NoAck(_ context.Context, _ []isb.Offset) {} - -func (r *KafkaSource) Start() <-chan struct{} { +func (r *kafkaSource) Start() <-chan struct{} { client, err := sarama.NewClient(r.brokers, r.config) if err != nil { r.logger.Panicw("Failed to create sarama client", zap.Error(err)) @@ -261,20 +209,20 @@ func (r *KafkaSource) Start() <-chan struct{} { return r.forwarder.Start() } -func (r *KafkaSource) Stop() { +func (r *kafkaSource) Stop() { r.logger.Info("Stopping kafka reader...") // stop the forwarder r.forwarder.Stop() } -func (r *KafkaSource) ForceStop() { +func (r *kafkaSource) ForceStop() { // ForceStop means everything has to stop. // don't wait for anything. r.forwarder.ForceStop() r.Stop() } -func (r *KafkaSource) Close() error { +func (r *kafkaSource) Close() error { r.logger.Info("Closing kafka reader...") // finally, shut down the client r.cancelFn() @@ -289,7 +237,7 @@ func (r *KafkaSource) Close() error { return nil } -func (r *KafkaSource) Pending(_ context.Context) (int64, error) { +func (r *kafkaSource) Pending(_ context.Context) (int64, error) { if r.adminClient == nil || r.saramaClient == nil { return isb.PendingNotAvailable, nil } @@ -324,35 +272,31 @@ func (r *KafkaSource) Pending(_ context.Context) (int64, error) { return totalPending, nil } -// NewKafkaSource returns a KafkaSource reader based on Kafka Consumer Group. +// NewKafkaSource returns a kafkaSource reader based on Kafka Consumer Group. func NewKafkaSource( vertexInstance *dfv1.VertexInstance, writers map[string][]isb.BufferWriter, fsd forwarder.ToWhichStepDecider, transformerApplier applier.SourceTransformApplier, - fetchWM fetch.Fetcher, + fetchWM fetch.SourceFetcher, toVertexPublisherStores map[string]store.WatermarkStore, publishWMStores store.WatermarkStore, idleManager wmb.IdleManager, - opts ...Option) (*KafkaSource, error) { + opts ...Option) (sourcer.Sourcer, error) { source := vertexInstance.Vertex.Spec.Source.Kafka - kafkaSource := &KafkaSource{ - vertexName: vertexInstance.Vertex.Spec.Name, - pipelineName: vertexInstance.Vertex.Spec.PipelineName, - topic: source.Topic, - brokers: source.Brokers, - readTimeout: 1 * time.Second, // default timeout - handlerBuffer: 100, // default buffer size for kafka reads - srcPublishWMStores: publishWMStores, - sourcePublishWMs: make(map[int32]publish.Publisher), - watermarkMaxDelay: vertexInstance.Vertex.Spec.Watermark.GetMaxDelay(), - lock: new(sync.RWMutex), - logger: logging.NewLogger(), // default logger + ks := &kafkaSource{ + vertexName: vertexInstance.Vertex.Spec.Name, + pipelineName: vertexInstance.Vertex.Spec.PipelineName, + topic: source.Topic, + brokers: source.Brokers, + readTimeout: 1 * time.Second, // default timeout + handlerBuffer: 100, // default buffer size for kafka reads + logger: logging.NewLogger(), // default logger } for _, o := range opts { - if err := o(kafkaSource); err != nil { + if err := o(ks); err != nil { return nil, err } } @@ -380,38 +324,40 @@ func NewKafkaSource( } } - sarama.Logger = zap.NewStdLog(kafkaSource.logger.Desugar()) + sarama.Logger = zap.NewStdLog(ks.logger.Desugar()) // return errors from the underlying kafka client using the Errors channel config.Consumer.Return.Errors = true - kafkaSource.config = config + ks.config = config ctx, cancel := context.WithCancel(context.Background()) - kafkaSource.cancelFn = cancel - kafkaSource.lifecycleCtx = ctx + ks.cancelFn = cancel + ks.lifecycleCtx = ctx - kafkaSource.stopCh = make(chan struct{}) + ks.stopCh = make(chan struct{}) - handler := newConsumerHandler(kafkaSource.handlerBuffer) - kafkaSource.handler = handler + handler := newConsumerHandler(ks.handlerBuffer) + ks.handler = handler - forwardOpts := []sourceforward.Option{sourceforward.WithLogger(kafkaSource.logger)} + forwardOpts := []sourceforward.Option{sourceforward.WithLogger(ks.logger)} if x := vertexInstance.Vertex.Spec.Limits; x != nil { if x.ReadBatchSize != nil { forwardOpts = append(forwardOpts, sourceforward.WithReadBatchSize(int64(*x.ReadBatchSize))) } } - forwarder, err := sourceforward.NewDataForward(vertexInstance, kafkaSource, writers, fsd, transformerApplier, fetchWM, kafkaSource, toVertexPublisherStores, idleManager, forwardOpts...) + // create a source watermark publisher + sourceWmPublisher := publish.NewSourcePublish(ctx, ks.pipelineName, ks.vertexName, publishWMStores, publish.WithDelay(vertexInstance.Vertex.Spec.Watermark.GetMaxDelay())) + sourceForwarder, err := sourceforward.NewDataForward(vertexInstance, ks, writers, fsd, transformerApplier, fetchWM, sourceWmPublisher, toVertexPublisherStores, idleManager, forwardOpts...) if err != nil { - kafkaSource.logger.Errorw("Error instantiating the forwarder", zap.Error(err)) + ks.logger.Errorw("Error instantiating the sourceForwarder", zap.Error(err)) return nil, err } - kafkaSource.forwarder = forwarder - return kafkaSource, nil + ks.forwarder = sourceForwarder + return ks, nil } // refreshAdminClient refreshes the admin client -func (r *KafkaSource) refreshAdminClient() error { +func (r *kafkaSource) refreshAdminClient() error { if _, err := r.saramaClient.RefreshController(); err != nil { return fmt.Errorf("failed to refresh controller, %w", err) } @@ -435,7 +381,7 @@ func configFromOpts(yamlConfig string) (*sarama.Config, error) { return config, nil } -func (r *KafkaSource) startConsumer() { +func (r *kafkaSource) startConsumer() { client, err := sarama.NewConsumerGroup(r.brokers, r.groupName, r.config) r.logger.Infow("creating NewConsumerGroup", zap.String("topic", r.topic), zap.String("consumerGroupName", r.groupName), zap.Strings("brokers", r.brokers)) if err != nil { diff --git a/pkg/sources/kafka/reader_test.go b/pkg/sources/kafka/reader_test.go index 98de9771f4..f9d22cdf55 100644 --- a/pkg/sources/kafka/reader_test.go +++ b/pkg/sources/kafka/reader_test.go @@ -55,7 +55,7 @@ func TestNewKafkasource(t *testing.T) { Replica: 0, } publishWMStore, _ := store.BuildNoOpWatermarkStore() - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) toVertexWmStores := map[string]store.WatermarkStore{ "testVertex": publishWMStore, } @@ -65,14 +65,14 @@ func TestNewKafkasource(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, ks) - assert.Equal(t, "default", ks.groupName) + assert.Equal(t, "default", ks.(*kafkaSource).groupName) // config is all set and initialized correctly - assert.NotNil(t, ks.config) - assert.Equal(t, 100, ks.handlerBuffer) - assert.Equal(t, 100*time.Millisecond, ks.readTimeout) - assert.Equal(t, 100, cap(ks.handler.messages)) - assert.NotNil(t, ks.forwarder) + assert.NotNil(t, ks.(*kafkaSource).config) + assert.Equal(t, 100, ks.(*kafkaSource).handlerBuffer) + assert.Equal(t, 100*time.Millisecond, ks.(*kafkaSource).readTimeout) + assert.Equal(t, 100, cap(ks.(*kafkaSource).handler.messages)) + assert.NotNil(t, ks.(*kafkaSource).forwarder) } func TestGroupNameOverride(t *testing.T) { @@ -98,13 +98,13 @@ func TestGroupNameOverride(t *testing.T) { Replica: 0, } publishWMStore, _ := store.BuildNoOpWatermarkStore() - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) toVertexWmStores := map[string]store.WatermarkStore{ "testVertex": publishWMStore, } ks, _ := NewKafkaSource(vi, toBuffers, myForwardToAllTest{}, applier.Terminal, fetchWatermark, toVertexWmStores, publishWMStore, wmb.NewIdleManager(len(toBuffers)), WithLogger(logging.NewLogger()), WithBufferSize(100), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) - assert.Equal(t, "default", ks.groupName) + assert.Equal(t, "default", ks.(*kafkaSource).groupName) } @@ -131,13 +131,13 @@ func TestDefaultBufferSize(t *testing.T) { Replica: 0, } publishWMStore, _ := store.BuildNoOpWatermarkStore() - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) toVertexWmStores := map[string]store.WatermarkStore{ "testVertex": publishWMStore, } ks, _ := NewKafkaSource(vi, toBuffers, myForwardToAllTest{}, applier.Terminal, fetchWatermark, toVertexWmStores, publishWMStore, wmb.NewIdleManager(len(toBuffers)), WithLogger(logging.NewLogger()), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) - assert.Equal(t, 100, ks.handlerBuffer) + assert.Equal(t, 100, ks.(*kafkaSource).handlerBuffer) } @@ -164,12 +164,12 @@ func TestBufferSizeOverrides(t *testing.T) { Replica: 0, } publishWMStore, _ := store.BuildNoOpWatermarkStore() - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) toVertexWmStores := map[string]store.WatermarkStore{ "testVertex": publishWMStore, } ks, _ := NewKafkaSource(vi, toBuffers, myForwardToAllTest{}, applier.Terminal, fetchWatermark, toVertexWmStores, publishWMStore, wmb.NewIdleManager(len(toBuffers)), WithLogger(logging.NewLogger()), WithBufferSize(110), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) - assert.Equal(t, 110, ks.handlerBuffer) + assert.Equal(t, 110, ks.(*kafkaSource).handlerBuffer) } diff --git a/pkg/sources/nats/nats.go b/pkg/sources/nats/nats.go index 0d360666d9..d0e2beec5e 100644 --- a/pkg/sources/nats/nats.go +++ b/pkg/sources/nats/nats.go @@ -33,7 +33,7 @@ import ( sharedutil "github.com/numaproj/numaflow/pkg/shared/util" sourceforward "github.com/numaproj/numaflow/pkg/sources/forward" "github.com/numaproj/numaflow/pkg/sources/forward/applier" - "github.com/numaproj/numaflow/pkg/watermark/entity" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/watermark/fetch" "github.com/numaproj/numaflow/pkg/watermark/publish" "github.com/numaproj/numaflow/pkg/watermark/store" @@ -41,21 +41,17 @@ import ( ) type natsSource struct { - name string - pipelineName string - logger *zap.SugaredLogger - - natsConn *natslib.Conn - sub *natslib.Subscription - - bufferSize int - messages chan *isb.ReadMessage - readTimeout time.Duration - - cancelFn context.CancelFunc - forwarder *sourceforward.DataForward - // source watermark publisher - sourcePublishWM publish.Publisher + vertexName string + pipelineName string + vertexReplica int32 + logger *zap.SugaredLogger + natsConn *natslib.Conn + sub *natslib.Subscription + bufferSize int + messages chan *isb.ReadMessage + readTimeout time.Duration + cancelFn context.CancelFunc + forwarder *sourceforward.DataForward } func New( @@ -63,17 +59,18 @@ func New( writers map[string][]isb.BufferWriter, fsd forwarder.ToWhichStepDecider, transformerApplier applier.SourceTransformApplier, - fetchWM fetch.Fetcher, + fetchWM fetch.SourceFetcher, toVertexPublisherStores map[string]store.WatermarkStore, publishWMStores store.WatermarkStore, idleManager wmb.IdleManager, - opts ...Option) (*natsSource, error) { + opts ...Option) (sourcer.Sourcer, error) { n := &natsSource{ - name: vertexInstance.Vertex.Spec.Name, - pipelineName: vertexInstance.Vertex.Spec.PipelineName, - bufferSize: 1000, // default size - readTimeout: 1 * time.Second, // default timeout + vertexName: vertexInstance.Vertex.Spec.Name, + pipelineName: vertexInstance.Vertex.Spec.PipelineName, + vertexReplica: vertexInstance.Replica, + bufferSize: 1000, // default size + readTimeout: 1 * time.Second, // default timeout } for _, o := range opts { if err := o(n); err != nil { @@ -91,25 +88,27 @@ func New( forwardOpts = append(forwardOpts, sourceforward.WithReadBatchSize(int64(*x.ReadBatchSize))) } } - forwarder, err := sourceforward.NewDataForward(vertexInstance, n, writers, fsd, transformerApplier, fetchWM, n, toVertexPublisherStores, idleManager, forwardOpts...) + + ctx, cancel := context.WithCancel(context.Background()) + n.cancelFn = cancel + + // create a source watermark publisher + sourceWmPublisher := publish.NewSourcePublish(ctx, n.pipelineName, n.vertexName, + publishWMStores, publish.WithDelay(vertexInstance.Vertex.Spec.Watermark.GetMaxDelay()), publish.WithDefaultPartitionIdx(vertexInstance.Replica)) + + sourceForwarder, err := sourceforward.NewDataForward(vertexInstance, n, writers, fsd, transformerApplier, fetchWM, sourceWmPublisher, toVertexPublisherStores, idleManager, forwardOpts...) if err != nil { - n.logger.Errorw("Error instantiating the forwarder", zap.Error(err)) + n.logger.Errorw("Error instantiating the sourceForwarder", zap.Error(err)) return nil, err } - n.forwarder = forwarder - ctx, cancel := context.WithCancel(context.Background()) - n.cancelFn = cancel - entityName := fmt.Sprintf("%s-%d", vertexInstance.Vertex.Name, vertexInstance.Replica) - processorEntity := entity.NewProcessorEntity(entityName) - // toVertexPartitionCount is 1 because we publish watermarks within the source itself. - n.sourcePublishWM = publish.NewPublish(ctx, processorEntity, publishWMStores, 1, publish.IsSource(), publish.WithDelay(vertexInstance.Vertex.Spec.Watermark.GetMaxDelay())) + n.forwarder = sourceForwarder source := vertexInstance.Vertex.Spec.Source.Nats opt := []natslib.Option{ natslib.MaxReconnects(-1), natslib.ReconnectWait(3 * time.Second), - natslib.DisconnectHandler(func(c *natslib.Conn) { - n.logger.Info("Nats disconnected") + natslib.DisconnectErrHandler(func(c *natslib.Conn, err error) { + n.logger.Errorw("Nats disconnected", zap.Error(err)) }), natslib.ReconnectHandler(func(c *natslib.Conn) { n.logger.Info("Nats reconnected") @@ -213,13 +212,12 @@ func WithReadTimeout(t time.Duration) Option { } func (ns *natsSource) GetName() string { - return ns.name + return ns.vertexName } -// GetPartitionIdx returns the partition number for the source vertex buffer -// Source is like a buffer with only one partition. So, we always return 0 -func (ns *natsSource) GetPartitionIdx() int32 { - return 0 +// Partitions returns the partitions associated with this source. +func (ns *natsSource) Partitions() []int32 { + return []int32{ns.vertexReplica} } func (ns *natsSource) Read(_ context.Context, count int64) ([]*isb.ReadMessage, error) { @@ -229,7 +227,7 @@ loop: for i := int64(0); i < count; i++ { select { case m := <-ns.messages: - natsSourceReadCount.With(map[string]string{metrics.LabelVertex: ns.name, metrics.LabelPipeline: ns.pipelineName}).Inc() + natsSourceReadCount.With(map[string]string{metrics.LabelVertex: ns.vertexName, metrics.LabelPipeline: ns.pipelineName}).Inc() msgs = append(msgs, m) case <-timeout: ns.logger.Debugw("Timed out waiting for messages to read.", zap.Duration("waited", ns.readTimeout), zap.Int("read", len(msgs))) @@ -244,25 +242,10 @@ func (ns *natsSource) Pending(_ context.Context) (int64, error) { return isb.PendingNotAvailable, nil } -func (ns *natsSource) PublishSourceWatermarks(msgs []*isb.ReadMessage) { - var oldest time.Time - for _, m := range msgs { - if oldest.IsZero() || m.EventTime.Before(oldest) { - oldest = m.EventTime - } - } - if len(msgs) > 0 && !oldest.IsZero() { - // toVertexPartitionIdx is 0 because we publish watermarks within the source itself. - ns.sourcePublishWM.PublishWatermark(wmb.Watermark(oldest), nil, 0) // Source publisher does not care about the offset - } -} - func (ns *natsSource) Ack(_ context.Context, offsets []isb.Offset) []error { return make([]error, len(offsets)) } -func (ns *natsSource) NoAck(_ context.Context, _ []isb.Offset) {} - func (ns *natsSource) Close() error { ns.logger.Info("Shutting down nats source server...") ns.cancelFn() diff --git a/pkg/sources/nats/nats_test.go b/pkg/sources/nats/nats_test.go index c5107b2040..14ea0fbee5 100644 --- a/pkg/sources/nats/nats_test.go +++ b/pkg/sources/nats/nats_test.go @@ -31,6 +31,7 @@ import ( "github.com/numaproj/numaflow/pkg/isb/stores/simplebuffer" natstest "github.com/numaproj/numaflow/pkg/shared/clients/nats/test" "github.com/numaproj/numaflow/pkg/sources/forward/applier" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/watermark/generic" "github.com/numaproj/numaflow/pkg/watermark/store" "github.com/numaproj/numaflow/pkg/watermark/wmb" @@ -70,7 +71,7 @@ func testVertex(t *testing.T, url, subject, queue string, hostname string, repli return vi } -func newInstance(t *testing.T, vi *dfv1.VertexInstance) (*natsSource, error) { +func newInstance(t *testing.T, vi *dfv1.VertexInstance) (sourcer.Sourcer, error) { t.Helper() dest := simplebuffer.NewInMemoryBuffer("test", 100, 0) toBuffers := map[string][]isb.BufferWriter{ @@ -78,7 +79,7 @@ func newInstance(t *testing.T, vi *dfv1.VertexInstance) (*natsSource, error) { } publishWMStores, _ := store.BuildNoOpWatermarkStore() - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressorsFromBufferMap(map[string][]isb.BufferWriter{}) toVertexWmStores := map[string]store.WatermarkStore{ "testVertex": publishWMStores, } diff --git a/pkg/sources/source.go b/pkg/sources/source.go index 284e4c32a1..b2c1b50e35 100644 --- a/pkg/sources/source.go +++ b/pkg/sources/source.go @@ -43,6 +43,7 @@ import ( "github.com/numaproj/numaflow/pkg/sources/http" "github.com/numaproj/numaflow/pkg/sources/kafka" "github.com/numaproj/numaflow/pkg/sources/nats" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/sources/transformer" "github.com/numaproj/numaflow/pkg/sources/udsource" "github.com/numaproj/numaflow/pkg/watermark/fetch" @@ -65,7 +66,7 @@ func (sp *SourceProcessor) Start(ctx context.Context) error { log = logging.FromContext(ctx) writersMap = make(map[string][]isb.BufferWriter) sdkClient sourcetransformer.Client - sourcer Sourcer + source sourcer.Sourcer readyCheckers []metrics.HealthChecker idleManager wmb.IdleManager ) @@ -82,7 +83,7 @@ func (sp *SourceProcessor) Start(ctx context.Context) error { // watermark variables no-op initialization // create a no op fetcher - fetchWatermark, _ := generic.BuildNoOpWatermarkProgressorsFromBufferList(sp.VertexInstance.Vertex.GetToBuffers()) + fetchWatermark, _ := generic.BuildNoOpSourceWatermarkProgressors(sp.VertexInstance.Vertex.GetToBuffers()) // create no op publisher stores for _, e := range sp.VertexInstance.Vertex.Spec.ToEdges { toVertexWatermarkStores[e.To], _ = store.BuildNoOpWatermarkStore() @@ -226,15 +227,15 @@ func (sp *SourceProcessor) Start(ctx context.Context) error { } readyCheckers = append(readyCheckers, transformerGRPCClient) - sourcer, err = sp.getSourcer(writersMap, sp.getTransformerGoWhereDecider(shuffleFuncMap), transformerGRPCClient, udsGRPCClient, fetchWatermark, toVertexWatermarkStores, sourcePublisherStores, idleManager, log) + source, err = sp.getSourcer(writersMap, sp.getTransformerGoWhereDecider(shuffleFuncMap), transformerGRPCClient, udsGRPCClient, fetchWatermark, toVertexWatermarkStores, sourcePublisherStores, idleManager, log) } else { - sourcer, err = sp.getSourcer(writersMap, sp.getSourceGoWhereDecider(shuffleFuncMap), applier.Terminal, udsGRPCClient, fetchWatermark, toVertexWatermarkStores, sourcePublisherStores, idleManager, log) + source, err = sp.getSourcer(writersMap, sp.getSourceGoWhereDecider(shuffleFuncMap), applier.Terminal, udsGRPCClient, fetchWatermark, toVertexWatermarkStores, sourcePublisherStores, idleManager, log) } if err != nil { - return fmt.Errorf("failed to find a sourcer, error: %w", err) + return fmt.Errorf("failed to find a source, error: %w", err) } log.Infow("Start processing source messages", zap.String("isbs", string(sp.ISBSvcType)), zap.Any("to", sp.VertexInstance.Vertex.GetToBuffers())) - stopped := sourcer.Start() + stopped := source.Start() wg := &sync.WaitGroup{} wg.Add(1) go func() { @@ -246,7 +247,7 @@ func (sp *SourceProcessor) Start(ctx context.Context) error { } }() - metricsOpts := metrics.NewMetricsOptions(ctx, sp.VertexInstance.Vertex, readyCheckers, []isb.BufferReader{sourcer}) + metricsOpts := metrics.NewMetricsOptions(ctx, sp.VertexInstance.Vertex, readyCheckers, []isb.LagReader{source}) ms := metrics.NewMetricsServer(sp.VertexInstance.Vertex, metricsOpts...) if shutdown, err := ms.Start(ctx); err != nil { return fmt.Errorf("failed to start metrics server, error: %w", err) @@ -255,7 +256,7 @@ func (sp *SourceProcessor) Start(ctx context.Context) error { } <-ctx.Done() log.Info("SIGTERM, exiting...") - sourcer.Stop() + source.Stop() wg.Wait() // close all the source wm stores @@ -281,11 +282,11 @@ func (sp *SourceProcessor) getSourcer( fsd forwarder.ToWhichStepDecider, transformerApplier applier.SourceTransformApplier, udsGRPCClient *udsource.GRPCBasedUDSource, - fetchWM fetch.Fetcher, + fetchWM fetch.SourceFetcher, toVertexPublisherStores map[string]store.WatermarkStore, publishWMStores store.WatermarkStore, idleManager wmb.IdleManager, - logger *zap.SugaredLogger) (Sourcer, error) { + logger *zap.SugaredLogger) (sourcer.Sourcer, error) { src := sp.VertexInstance.Vertex.Spec.Source if x := src.UDSource; x != nil && udsGRPCClient != nil { diff --git a/pkg/sources/sourcer.go b/pkg/sources/sourcer.go deleted file mode 100644 index e532e2c247..0000000000 --- a/pkg/sources/sourcer.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2022 The Numaproj Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sources - -import ( - "github.com/numaproj/numaflow/pkg/forwarder" - "github.com/numaproj/numaflow/pkg/isb" -) - -// Sourcer interface provides an isb.BufferReader abstraction over the underlying data source. -// This is intended to be consumed by a connector like isb.forward -type Sourcer interface { - isb.BufferReader - forwarder.StarterStopper - isb.LagReader - isb.SourceWatermarkPublisher -} diff --git a/pkg/sources/sourcer/sourcer.go b/pkg/sources/sourcer/sourcer.go new file mode 100644 index 0000000000..89983f965d --- /dev/null +++ b/pkg/sources/sourcer/sourcer.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The Numaproj Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sourcer + +import ( + "context" + "io" + + "github.com/numaproj/numaflow/pkg/forwarder" + "github.com/numaproj/numaflow/pkg/isb" +) + +type SourceReader interface { + io.Closer + // GetName returns the name of the source. + GetName() string + // Read reads a chunk of messages and returns at the first occurrence of an error. Error does not indicate that the + // array of result is empty, the callee should process all the elements in the array even if the error is set. Read + // will not mark the message in the buffer as "READ" if the read for that index is erring. + // There is a chance that we have read the message and the container got forcefully terminated before processing. To provide + // at-least-once semantics for reading, during the restart we will have to reprocess all unacknowledged messages. + Read(context.Context, int64) ([]*isb.ReadMessage, error) + // Ack acknowledges an array of offset. + Ack(context.Context, []isb.Offset) []error + // Partitions returns the partitions of the source. This is used by the forwarder to determine to which partition + // idle watermarks should be published. Partition assignment to a pod is dynamic, so this method may return different + // partitions at different times. (Example - Kafka, every time topic rebalancing happens, the partitions gets updated) + Partitions() []int32 +} + +// Sourcer interface provides an isb.BufferReader abstraction over the underlying data source. +// This is intended to be consumed by a connector like isb.forward +type Sourcer interface { + SourceReader + forwarder.StarterStopper + isb.LagReader +} diff --git a/pkg/sources/udsource/user_defined_source.go b/pkg/sources/udsource/user_defined_source.go index 279bc55be1..f2fadaceaf 100644 --- a/pkg/sources/udsource/user_defined_source.go +++ b/pkg/sources/udsource/user_defined_source.go @@ -18,8 +18,6 @@ package udsource import ( "context" - "fmt" - "sync" "time" "go.uber.org/zap" @@ -30,7 +28,7 @@ import ( "github.com/numaproj/numaflow/pkg/shared/logging" sourceforward "github.com/numaproj/numaflow/pkg/sources/forward" "github.com/numaproj/numaflow/pkg/sources/forward/applier" - "github.com/numaproj/numaflow/pkg/watermark/entity" + "github.com/numaproj/numaflow/pkg/sources/sourcer" "github.com/numaproj/numaflow/pkg/watermark/fetch" "github.com/numaproj/numaflow/pkg/watermark/publish" "github.com/numaproj/numaflow/pkg/watermark/store" @@ -56,28 +54,16 @@ func WithReadTimeout(t time.Duration) Option { } type userDefinedSource struct { - // name of the user-defined source vertex - vertexName string - // name of the pipeline - pipelineName string - // sourceApplier applies the user-defined source functions - sourceApplier *GRPCBasedUDSource - // forwarder writes the source data to destination - forwarder *sourceforward.DataForward - // context cancel function - cancelFn context.CancelFunc - // source watermark publishers for different partitions - srcWMPublishers map[int32]publish.Publisher - // source watermark publisher stores - srcPublishWMStores store.WatermarkStore - // lifecycleCtx context is used to control the lifecycle of this source. - lifecycleCtx context.Context - // read timeout for the source - readTimeout time.Duration - // logger - logger *zap.SugaredLogger - // a lock to protect srcWMPublishers - lock *sync.RWMutex + vertexName string // name of the user-defined source vertex + pipelineName string // name of the pipeline + sourceApplier *GRPCBasedUDSource // sourceApplier applies the user-defined source functions + forwarder *sourceforward.DataForward // forwarder writes the source data to destination + cancelFn context.CancelFunc // context cancel function + srcPublishWMStores store.WatermarkStore // source watermark publisher stores + lifecycleCtx context.Context // lifecycleCtx context is used to control the lifecycle of this source. + readTimeout time.Duration // read timeout for the source + partitions map[int32]struct{} // partitions of the source + logger *zap.SugaredLogger } func New( @@ -86,19 +72,20 @@ func New( fsd forwarder.ToWhichStepDecider, transformer applier.SourceTransformApplier, sourceApplier *GRPCBasedUDSource, - fetchWM fetch.Fetcher, + fetchWM fetch.SourceFetcher, toVertexPublisherStores map[string]store.WatermarkStore, publishWMStores store.WatermarkStore, idleManager wmb.IdleManager, - opts ...Option) (*userDefinedSource, error) { + opts ...Option) (sourcer.Sourcer, error) { + + var err error u := &userDefinedSource{ vertexName: vertexInstance.Vertex.Spec.Name, pipelineName: vertexInstance.Vertex.Spec.PipelineName, sourceApplier: sourceApplier, srcPublishWMStores: publishWMStores, - srcWMPublishers: make(map[int32]publish.Publisher), - lock: new(sync.RWMutex), + partitions: make(map[int32]struct{}), logger: logging.NewLogger(), // default logger } for _, opt := range opts { @@ -113,15 +100,19 @@ func New( forwardOpts = append(forwardOpts, sourceforward.WithReadBatchSize(int64(*x.ReadBatchSize))) } } - var err error - u.forwarder, err = sourceforward.NewDataForward(vertexInstance, u, writers, fsd, transformer, fetchWM, u, toVertexPublisherStores, idleManager, forwardOpts...) + + ctx, cancel := context.WithCancel(context.Background()) + u.cancelFn = cancel + u.lifecycleCtx = ctx + + // create a source watermark publisher + sourceWmPublisher := publish.NewSourcePublish(ctx, u.pipelineName, u.vertexName, publishWMStores, publish.WithDelay(vertexInstance.Vertex.Spec.Watermark.GetMaxDelay())) + + u.forwarder, err = sourceforward.NewDataForward(vertexInstance, u, writers, fsd, transformer, fetchWM, sourceWmPublisher, toVertexPublisherStores, idleManager, forwardOpts...) if err != nil { u.logger.Errorw("Error instantiating the forwarder", zap.Error(err)) return nil, err } - ctx, cancel := context.WithCancel(context.Background()) - u.cancelFn = cancel - u.lifecycleCtx = ctx return u, nil } @@ -130,15 +121,28 @@ func (u *userDefinedSource) GetName() string { return u.vertexName } -// GetPartitionIdx returns the partition number for the user-defined source. -// Source is like a buffer with only one partition. So, we always return 0 -func (u *userDefinedSource) GetPartitionIdx() int32 { - return 0 +// Partitions returns the partitions of the user-defined source +func (u *userDefinedSource) Partitions() []int32 { + partitions := make([]int32, 0, len(u.partitions)) + for partition := range u.partitions { + partitions = append(partitions, partition) + } + return partitions } -// Read reads the messages from the user-defined source +// Read reads the messages from the user-defined source, tracks the partitions from which the messages are read +// tracked partitions are used to determine the partitions to which the watermarks should be published func (u *userDefinedSource) Read(ctx context.Context, count int64) ([]*isb.ReadMessage, error) { - return u.sourceApplier.ApplyReadFn(ctx, count, u.readTimeout) + readMessages, err := u.sourceApplier.ApplyReadFn(ctx, count, u.readTimeout) + if err != nil { + return nil, err + } + for _, msg := range readMessages { + if _, ok := u.partitions[msg.ReadOffset.PartitionIdx()]; !ok { + u.partitions[msg.ReadOffset.PartitionIdx()] = struct{}{} + } + } + return readMessages, nil } // Ack acknowledges the messages from the user-defined source @@ -152,10 +156,6 @@ func (u *userDefinedSource) Pending(ctx context.Context) (int64, error) { return u.sourceApplier.ApplyPendingFn(ctx) } -func (u *userDefinedSource) NoAck(_ context.Context, _ []isb.Offset) { - panic("User defined source does not support NoAck") -} - func (u *userDefinedSource) Close() error { u.logger.Info("Shutting down user-defined source...") u.cancelFn() @@ -177,34 +177,3 @@ func (u *userDefinedSource) Start() <-chan struct{} { u.logger.Info("Starting user-defined source...") return u.forwarder.Start() } - -func (u *userDefinedSource) PublishSourceWatermarks(msgs []*isb.ReadMessage) { - // oldestTimestamps stores the latest timestamps for different partitions - oldestTimestamps := make(map[int32]time.Time) - for _, m := range msgs { - // Get latest timestamps for different partitions - if t, ok := oldestTimestamps[m.ReadOffset.PartitionIdx()]; !ok || m.EventTime.Before(t) { - oldestTimestamps[m.ReadOffset.PartitionIdx()] = m.EventTime - } - } - for p, t := range oldestTimestamps { - publisher := u.loadSourceWatermarkPublisher(p) - // toVertexPartitionIdx is 0 because we publish watermarks within the source itself. - publisher.PublishWatermark(wmb.Watermark(t), nil, 0) // Source publisher does not care about the offset - } -} - -// loadSourceWatermarkPublisher does a lazy load on the watermark publisher -func (u *userDefinedSource) loadSourceWatermarkPublisher(partitionID int32) publish.Publisher { - u.lock.Lock() - defer u.lock.Unlock() - if p, ok := u.srcWMPublishers[partitionID]; ok { - return p - } - entityName := fmt.Sprintf("%s-%s-%d", u.pipelineName, u.vertexName, partitionID) - processorEntity := entity.NewProcessorEntity(entityName) - // toVertexPartitionCount is 1 because we publish watermarks within the source itself. - sourcePublishWM := publish.NewPublish(u.lifecycleCtx, processorEntity, u.srcPublishWMStores, 1, publish.IsSource()) - u.srcWMPublishers[partitionID] = sourcePublishWM - return sourcePublishWM -} diff --git a/pkg/sources/udsource/utils/offset.go b/pkg/sources/udsource/utils/offset.go index 50df7b746b..e83a3adeca 100644 --- a/pkg/sources/udsource/utils/offset.go +++ b/pkg/sources/udsource/utils/offset.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Numaproj Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package utils import ( diff --git a/pkg/sources/udsource/utils/offset_test.go b/pkg/sources/udsource/utils/offset_test.go index a083f5a2c1..9586d9b6b5 100644 --- a/pkg/sources/udsource/utils/offset_test.go +++ b/pkg/sources/udsource/utils/offset_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Numaproj Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package utils import ( diff --git a/pkg/udf/forward/forward_test.go b/pkg/udf/forward/forward_test.go index 73dcb68b7f..820bee2dd5 100644 --- a/pkg/udf/forward/forward_test.go +++ b/pkg/udf/forward/forward_test.go @@ -1312,7 +1312,7 @@ func (f mySourceForwardTest) ApplyMapStream(ctx context.Context, message *isb.Re }(ctx, message, writeMessageCh) } -// TestSourceWatermarkPublisher is a dummy implementation of isb.SourceWatermarkPublisher interface +// TestSourceWatermarkPublisher is a dummy implementation of isb.SourcePublisher interface type TestSourceWatermarkPublisher struct { } diff --git a/pkg/udf/map_udf.go b/pkg/udf/map_udf.go index c00d25afbb..cff33005d3 100644 --- a/pkg/udf/map_udf.go +++ b/pkg/udf/map_udf.go @@ -259,12 +259,17 @@ func (u *MapUDFProcessor) Start(ctx context.Context) error { log.Info("Exited for partition...", zap.String("partition", fromBufferPartitionName)) }(bufferPartition, forwarder) } + // create lag readers from buffer readers + var lagReaders []isb.LagReader + for _, reader := range readers { + lagReaders = append(lagReaders, reader) + } var metricsOpts []metrics.Option if enableMapUdfStream { - metricsOpts = metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{mapStreamHandler}, readers) + metricsOpts = metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{mapStreamHandler}, lagReaders) } else { - metricsOpts = metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{mapHandler}, readers) + metricsOpts = metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{mapHandler}, lagReaders) } ms := metrics.NewMetricsServer(u.VertexInstance.Vertex, metricsOpts...) diff --git a/pkg/udf/reduce_udf.go b/pkg/udf/reduce_udf.go index d18fe758c8..9bdefcaf00 100644 --- a/pkg/udf/reduce_udf.go +++ b/pkg/udf/reduce_udf.go @@ -221,8 +221,14 @@ func (u *ReduceUDFProcessor) Start(ctx context.Context) error { }() log.Infow("Start processing reduce udf messages", zap.String("isbsvc", string(u.ISBSvcType)), zap.String("from", fromBuffer)) + // create lag readers from buffer readers + var lagReaders []isb.LagReader + for _, reader := range readers { + lagReaders = append(lagReaders, reader) + } + // start metrics server - metricsOpts := metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{reduceHandler}, readers) + metricsOpts := metrics.NewMetricsOptions(ctx, u.VertexInstance.Vertex, []metrics.HealthChecker{reduceHandler}, lagReaders) ms := metrics.NewMetricsServer(u.VertexInstance.Vertex, metricsOpts...) if shutdown, err := ms.Start(ctx); err != nil { return fmt.Errorf("failed to start metrics server, error: %w", err) diff --git a/pkg/watermark/fetch/interface.go b/pkg/watermark/fetch/interface.go index 62e4495b21..fddf287b2b 100644 --- a/pkg/watermark/fetch/interface.go +++ b/pkg/watermark/fetch/interface.go @@ -29,8 +29,15 @@ type Fetcher interface { ComputeHeadIdleWMB(fromPartitionIdx int32) wmb.WMB } -// UXFetcher computes the watermark for Vn. -type UXFetcher interface { +// SourceFetcher fetches watermark data for source vertex. +type SourceFetcher interface { + // ComputeWatermark computes the watermark, it will return the minimum of all the watermarks of the processors. + ComputeWatermark() wmb.Watermark + HeadFetcher +} + +// HeadFetcher computes the watermark for Vn. +type HeadFetcher interface { // ComputeHeadWatermark computes a valid head watermark for the given partition ComputeHeadWatermark(fromPartitionIdx int32) wmb.Watermark } diff --git a/pkg/watermark/fetch/source_fetcher.go b/pkg/watermark/fetch/source_fetcher.go index c6caa9ca75..1605c55706 100644 --- a/pkg/watermark/fetch/source_fetcher.go +++ b/pkg/watermark/fetch/source_fetcher.go @@ -25,7 +25,6 @@ import ( "go.uber.org/zap" - "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/shared/logging" "github.com/numaproj/numaflow/pkg/watermark/store" "github.com/numaproj/numaflow/pkg/watermark/wmb" @@ -39,7 +38,7 @@ type sourceFetcher struct { // NewSourceFetcher returns a new source fetcher, pm has the details about the processors responsible for writing to the // buckets of the source buffer. -func NewSourceFetcher(ctx context.Context, store store.WatermarkStore, opts ...Option) Fetcher { +func NewSourceFetcher(ctx context.Context, store store.WatermarkStore, opts ...Option) SourceFetcher { log := logging.FromContext(ctx) log.Info("Creating a new source watermark fetcher") manager := newProcessorManager(ctx, store, 1, opts...) @@ -49,7 +48,7 @@ func NewSourceFetcher(ctx context.Context, store store.WatermarkStore, opts ...O } } -func (e *sourceFetcher) ComputeWatermark(_ isb.Offset, _ int32) wmb.Watermark { +func (e *sourceFetcher) ComputeWatermark() wmb.Watermark { return e.getWatermark() } @@ -98,9 +97,3 @@ func (e *sourceFetcher) ComputeHeadWatermark(fromPartitionIdx int32) wmb.Waterma } return wmb.Watermark(time.UnixMilli(epoch)) } - -// ComputeHeadIdleWMB returns the latest idle WMB among all processors -func (e *sourceFetcher) ComputeHeadIdleWMB(int32) wmb.WMB { - // TODO: what would this be... - return wmb.WMB{} -} diff --git a/pkg/watermark/generic/noop.go b/pkg/watermark/generic/noop.go index e9d48d71d1..87b9795ee7 100644 --- a/pkg/watermark/generic/noop.go +++ b/pkg/watermark/generic/noop.go @@ -28,6 +28,7 @@ import ( type NoOpWMProgressor struct { } +// if the struct implements SourceFetcher interface, it implicitly implements Fetcher. var _ fetch.Fetcher = (*NoOpWMProgressor)(nil) var _ publish.Publisher = (*NoOpWMProgressor)(nil) @@ -68,6 +69,42 @@ func (n NoOpWMProgressor) Close() error { return nil } +type NoOpSourceWMProgressor struct { +} + +var _ fetch.SourceFetcher = (*NoOpSourceWMProgressor)(nil) +var _ publish.Publisher = (*NoOpSourceWMProgressor)(nil) + +func NewNoOpSourceWMProgressor() *NoOpSourceWMProgressor { + return &NoOpSourceWMProgressor{} +} + +// Close stops the no-op progressor. +func (n NoOpSourceWMProgressor) Close() error { + return nil +} + +// PublishWatermark does a no-op watermark publish. +func (n NoOpSourceWMProgressor) PublishWatermark(_ wmb.Watermark, _ isb.Offset, _ int32) {} + +// PublishIdleWatermark does a no-op idle watermark publish. +func (n NoOpSourceWMProgressor) PublishIdleWatermark(_ wmb.Watermark, _ isb.Offset, _ int32) {} + +// GetLatestWatermark returns the default watermark as the latest watermark. +func (n NoOpSourceWMProgressor) GetLatestWatermark() wmb.Watermark { + return wmb.Watermark{} +} + +// ComputeWatermark returns the default watermark. +func (n NoOpSourceWMProgressor) ComputeWatermark() wmb.Watermark { + return wmb.Watermark{} +} + +// ComputeHeadWatermark returns the default head watermark. +func (n NoOpSourceWMProgressor) ComputeHeadWatermark(int32) wmb.Watermark { + return wmb.Watermark{} +} + func BuildNoOpWatermarkProgressorsFromBufferList(toBuffers []string) (fetch.Fetcher, map[string]publish.Publisher) { fetchWatermark := NewNoOpWMProgressor() publishWatermark := make(map[string]publish.Publisher) @@ -77,6 +114,15 @@ func BuildNoOpWatermarkProgressorsFromBufferList(toBuffers []string) (fetch.Fetc return fetchWatermark, publishWatermark } +func BuildNoOpSourceWatermarkProgressors(toBuffers []string) (fetch.SourceFetcher, map[string]publish.Publisher) { + fetchWatermark := NewNoOpSourceWMProgressor() + publishWatermark := make(map[string]publish.Publisher) + for _, buffer := range toBuffers { + publishWatermark[buffer] = NewNoOpSourceWMProgressor() + } + return fetchWatermark, publishWatermark +} + func BuildNoOpWatermarkProgressorsFromBufferMap(bufferMap map[string][]isb.BufferWriter) (fetch.Fetcher, map[string]publish.Publisher) { fetchWatermark := NewNoOpWMProgressor() publishWatermark := make(map[string]publish.Publisher) @@ -85,3 +131,12 @@ func BuildNoOpWatermarkProgressorsFromBufferMap(bufferMap map[string][]isb.Buffe } return fetchWatermark, publishWatermark } + +func BuildNoOpSourceWatermarkProgressorsFromBufferMap(bufferMap map[string][]isb.BufferWriter) (fetch.SourceFetcher, map[string]publish.Publisher) { + fetchWatermark := NewNoOpSourceWMProgressor() + publishWatermark := make(map[string]publish.Publisher) + for buffName := range bufferMap { + publishWatermark[buffName] = NewNoOpSourceWMProgressor() + } + return fetchWatermark, publishWatermark +} diff --git a/pkg/watermark/publish/options.go b/pkg/watermark/publish/options.go index 3a27dd4211..4c3f69b683 100644 --- a/pkg/watermark/publish/options.go +++ b/pkg/watermark/publish/options.go @@ -34,6 +34,9 @@ type publishOptions struct { // Whether it is sink publisher or not // isSource and isSink should not be both true isSink bool + // partitionIdx is the default partition index + // It will be only used by source publisher + defaultPartitionIdx int32 } type PublishOption func(*publishOptions) @@ -70,3 +73,10 @@ func IsSink() PublishOption { opts.isSink = true } } + +// WithDefaultPartitionIdx sets the default partition index +func WithDefaultPartitionIdx(partitionIdx int32) PublishOption { + return func(opts *publishOptions) { + opts.defaultPartitionIdx = partitionIdx + } +} diff --git a/pkg/watermark/publish/publisher.go b/pkg/watermark/publish/publisher.go index 58a6b893cc..9b7db9cc89 100644 --- a/pkg/watermark/publish/publisher.go +++ b/pkg/watermark/publish/publisher.go @@ -126,6 +126,11 @@ func (p *publish) initialSetup() { // PublishWatermark publishes watermark and will retry until it can succeed. It will not publish if the new-watermark // is less than the current head watermark. func (p *publish) PublishWatermark(wm wmb.Watermark, offset isb.Offset, toVertexPartitionIdx int32) { + // if its a source, we need to add the delay to the watermark + if p.opts.isSource && p.opts.delay.Nanoseconds() > 0 && !time.Time(wm).IsZero() { + wm = wmb.Watermark(time.Time(wm).Add(-p.opts.delay)) + } + validWM, skipWM := p.validateWatermark(wm, toVertexPartitionIdx) if skipWM { return @@ -168,9 +173,6 @@ func (p *publish) PublishWatermark(wm wmb.Watermark, offset isb.Offset, toVertex // validateWatermark checks if the new watermark is greater than the head watermark, return true if yes, // otherwise, return false func (p *publish) validateWatermark(wm wmb.Watermark, toVertexPartitionIdx int32) (wmb.Watermark, bool) { - if p.opts.isSource && p.opts.delay.Nanoseconds() > 0 && !time.Time(wm).IsZero() { - wm = wmb.Watermark(time.Time(wm).Add(-p.opts.delay)) - } // update p.headWatermarks only if wm > p.headWatermarks headWM := p.GetHeadWM(toVertexPartitionIdx) if wm.AfterWatermark(headWM) { @@ -187,6 +189,8 @@ func (p *publish) validateWatermark(wm wmb.Watermark, toVertexPartitionIdx int32 } // PublishIdleWatermark publishes the idle watermark and will retry until it can succeed. +// while publishing idle watermark for source we don't add the delay because the idle watermark +// is the current watermark + the increment by value. // TODO: merge with PublishWatermark func (p *publish) PublishIdleWatermark(wm wmb.Watermark, offset isb.Offset, toVertexPartitionIdx int32) { var key = p.entity.GetName() diff --git a/pkg/watermark/publish/source_publisher.go b/pkg/watermark/publish/source_publisher.go new file mode 100644 index 0000000000..62f3e5950e --- /dev/null +++ b/pkg/watermark/publish/source_publisher.go @@ -0,0 +1,123 @@ +/* +Copyright 2022 The Numaproj Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package publish + +import ( + "context" + "fmt" + "time" + + "github.com/numaproj/numaflow/pkg/isb" + "github.com/numaproj/numaflow/pkg/watermark/entity" + "github.com/numaproj/numaflow/pkg/watermark/store" + "github.com/numaproj/numaflow/pkg/watermark/wmb" +) + +// SourcePublisher publishes source watermarks based on a list of isb.ReadMessage +// and also publishes idle watermarks. It internally calls the main WM Publisher to publish +// the watermark. The only difference is that in source we do not care about offsets in the first +// Publish within the source itself (huh? :-D). Also, when the source boots up, it has to load +// the watermark information so it can know about the global source WM state. +type SourcePublisher interface { + // PublishSourceWatermarks publishes source watermarks. + PublishSourceWatermarks([]*isb.ReadMessage) + // PublishIdleWatermarks publishes idle watermarks for the given partitions. + PublishIdleWatermarks(wm time.Time, partitions []int32) +} + +type sourcePublish struct { + ctx context.Context + pipelineName string + vertexName string + srcPublishWMStores store.WatermarkStore + sourcePublishWMs map[int32]Publisher + opts *publishOptions +} + +// NewSourcePublish returns a new source publisher. +func NewSourcePublish(ctx context.Context, pipelineName, vertexName string, srcPublishWMStores store.WatermarkStore, opts ...PublishOption) SourcePublisher { + options := &publishOptions{ + defaultPartitionIdx: -1, + delay: 0, + } + for _, opt := range opts { + opt(options) + } + + sp := &sourcePublish{ + ctx: ctx, + pipelineName: pipelineName, + vertexName: vertexName, + srcPublishWMStores: srcPublishWMStores, + sourcePublishWMs: make(map[int32]Publisher), + opts: options, + } + + // if defaultPartitionIdx is set, create a publisher for it + // will be used by http, nats and tickgen source whose partitionIdx is same + // as the vertex replica index + if sp.opts.defaultPartitionIdx != -1 { + entityName := fmt.Sprintf("%s-%s-%d", sp.pipelineName, sp.vertexName, sp.opts.defaultPartitionIdx) + processorEntity := entity.NewProcessorEntity(entityName) + // toVertexPartitionCount is 1 because we publish watermarks within the source itself. + sourcePublishWM := NewPublish(sp.ctx, processorEntity, sp.srcPublishWMStores, 1, IsSource(), WithDelay(sp.opts.delay)) + sp.sourcePublishWMs[sp.opts.defaultPartitionIdx] = sourcePublishWM + } + + return sp +} + +// PublishSourceWatermarks publishes source watermarks for a list of isb.ReadMessage. +// it publishes for the partitions to which the messages belong, it publishes the oldest timestamp +// seen for that partition in the list of messages. +// if a publisher for a partition doesn't exist, it creates one. +func (df *sourcePublish) PublishSourceWatermarks(readMessages []*isb.ReadMessage) { + // oldestTimestamps stores the latest timestamps for different partitions + oldestTimestamps := make(map[int32]time.Time) + for _, m := range readMessages { + // Get latest timestamps for different partitions + if t, ok := oldestTimestamps[m.ReadOffset.PartitionIdx()]; !ok || m.EventTime.Before(t) { + oldestTimestamps[m.ReadOffset.PartitionIdx()] = m.EventTime + } + } + for p, t := range oldestTimestamps { + publisher := df.loadSourceWatermarkPublisher(p) + // toVertexPartitionIdx is 0 because we publish watermarks within the source itself. + publisher.PublishWatermark(wmb.Watermark(t), nil, 0) // we don't care about the offset while publishing source watermark + } +} + +// PublishIdleWatermarks publishes idle watermarks for all partitions. +func (df *sourcePublish) PublishIdleWatermarks(wm time.Time, partitions []int32) { + for _, partitionId := range partitions { + publisher := df.loadSourceWatermarkPublisher(partitionId) + publisher.PublishIdleWatermark(wmb.Watermark(wm), nil, 0) // while publishing idle watermark at source, we don't care about the offset + } +} + +// loadSourceWatermarkPublisher does a lazy load on the watermark publisher +func (df *sourcePublish) loadSourceWatermarkPublisher(partitionID int32) Publisher { + if p, ok := df.sourcePublishWMs[partitionID]; ok { + return p + } + entityName := fmt.Sprintf("%s-%s-%d", df.pipelineName, df.vertexName, partitionID) + processorEntity := entity.NewProcessorEntity(entityName) + // toVertexPartitionCount is 1 because we publish watermarks within the source itself. + sourcePublishWM := NewPublish(df.ctx, processorEntity, df.srcPublishWMStores, 1, IsSource(), WithDelay(df.opts.delay)) + df.sourcePublishWMs[partitionID] = sourcePublishWM + return sourcePublishWM +} diff --git a/pkg/watermark/wmb/interface.go b/pkg/watermark/wmb/interface.go index 3f786fb8f6..b3ee913965 100644 --- a/pkg/watermark/wmb/interface.go +++ b/pkg/watermark/wmb/interface.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Numaproj Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package wmb import "github.com/numaproj/numaflow/pkg/isb" diff --git a/pkg/watermark/wmb/noop_idle_manager.go b/pkg/watermark/wmb/noop_idle_manager.go index 068f9f1c72..e0c68c7137 100644 --- a/pkg/watermark/wmb/noop_idle_manager.go +++ b/pkg/watermark/wmb/noop_idle_manager.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Numaproj Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package wmb import ( diff --git a/pkg/watermark/wmb/watermark.go b/pkg/watermark/wmb/watermark.go index d920e93e60..4655405424 100644 --- a/pkg/watermark/wmb/watermark.go +++ b/pkg/watermark/wmb/watermark.go @@ -50,3 +50,7 @@ func (w Watermark) Before(t time.Time) bool { func (w Watermark) BeforeWatermark(compare Watermark) bool { return w.Before(time.Time(compare)) } + +func (w Watermark) Add(t time.Duration) time.Time { + return time.Time(w).Add(t) +} diff --git a/test/idle-source-e2e/idle_source_test.go b/test/idle-source-e2e/idle_source_test.go new file mode 100644 index 0000000000..0bcaf9d818 --- /dev/null +++ b/test/idle-source-e2e/idle_source_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2022 The Numaproj Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package idle_source_e2e + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + . "github.com/numaproj/numaflow/test/fixtures" +) + +type IdleSourceSuite struct { + E2ESuite +} + +func (is *IdleSourceSuite) TestIdleKeyedReducePipeline() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + w := is.Given().Pipeline("@testdata/idle-source-reduce-pipeline.yaml"). + When(). + CreatePipelineAndWait() + defer w.DeletePipelineAndWait() + pipelineName := "idle-source" + + // wait for all the pods to come up + w.Expect().VertexPodsRunning() + + done := make(chan struct{}) + go func() { + // publish messages to source vertex, with event time starting from 60000 + startTime := 100 + for i := 0; true; i++ { + select { + case <-ctx.Done(): + return + case <-done: + return + default: + eventTime := strconv.Itoa(startTime + i*1000) + w.SendMessageTo(pipelineName, "in", NewHttpPostRequest().WithBody([]byte("1")).WithHeader("X-Numaflow-Event-Time", eventTime)). + SendMessageTo(pipelineName, "in", NewHttpPostRequest().WithBody([]byte("2")).WithHeader("X-Numaflow-Event-Time", eventTime)). + SendMessageTo(pipelineName, "in", NewHttpPostRequest().WithBody([]byte("3")).WithHeader("X-Numaflow-Event-Time", eventTime)) + time.Sleep(10 * time.Millisecond) + } + } + }() + + // since the key can be even or odd and the window duration is 10s + // the sum should be 20(for even) and 40(for odd) + w.Expect(). + SinkContains("sink", "40", WithTimeout(120*time.Second)). + SinkContains("sink", "20", WithTimeout(120*time.Second)) + done <- struct{}{} +} + +func TestReduceSuite(t *testing.T) { + suite.Run(t, new(IdleSourceSuite)) +} diff --git a/test/idle-source-e2e/testdata/idle-source-reduce-pipeline.yaml b/test/idle-source-e2e/testdata/idle-source-reduce-pipeline.yaml new file mode 100644 index 0000000000..b7410a1807 --- /dev/null +++ b/test/idle-source-e2e/testdata/idle-source-reduce-pipeline.yaml @@ -0,0 +1,55 @@ +apiVersion: numaflow.numaproj.io/v1alpha1 +kind: Pipeline +metadata: + name: idle-source +spec: + watermark: + idleSource: + threshold: 5s # The pipeline will be considered idle if the source has not emitted any data for given threshold value. + incrementBy: 3s # If source is found to be idle then increment the watermark by given incrementBy value. + stepInterval: 2s # If source is idling then publish the watermark only when step interval has passed. + limits: + readBatchSize: 50 + vertices: + - name: in + scale: + min: 2 + source: + http: {} + - name: atoi + scale: + min: 1 + udf: + container: + # Tell the input number is even or odd, see https://github.com/numaproj/numaflow-go/tree/main/pkg/mapper/examples/even_odd + image: quay.io/numaio/numaflow-go/map-even-odd:v0.5.0 + - name: compute-sum + partitions: 2 + udf: + container: + # Compute the sum, see https://github.com/numaproj/numaflow-go/tree/main/pkg/reducer/examples/sum + image: quay.io/numaio/numaflow-go/reduce-sum:v0.5.0 + groupBy: + window: + fixed: + length: 10s + keyed: true + storage: + persistentVolumeClaim: + volumeSize: 2Gi + accessMode: ReadWriteOnce + - name: sink + scale: + min: 1 + sink: + udsink: + container: + # A redis sink for e2e testing, see https://github.com/numaproj/numaflow-sinks/tree/main/redis-e2e-test-sink + image: quay.io/numaio/numaflow-sink/redis-e2e-test-sink:v0.5.0 + edges: + - from: in + to: atoi + - from: atoi + to: compute-sum + - from: compute-sum + to: sink \ No newline at end of file