merging main

Signed-off-by: veds-g <[email protected]>
numaproj · Oct 31, 2024 · 86c7ed0 · 86c7ed0
2 parents 75c6126 + 00a74df
commit 86c7ed0
Show file tree

Hide file tree

Showing 223 changed files with 27,872 additions and 8,530 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -1,14 +1,16 @@
 ignore:
-- "**/*.pb.go"
-- "**/*.pb.gw.go"
-- "**/*generated.go"
-- "**/*generated.deepcopy.go"
-- "**/*_test.go"
-- "pkg/client/.*"
-- "vendor/.*"
-- "test/.*"
-- "rust/**/error.rs"
-- "rust/numaflow-models/**" # ignore generated files
+  - "**/*.pb.go"
+  - "**/*.pb.gw.go"
+  - "**/*generated.go"
+  - "**/*generated.deepcopy.go"
+  - "**/*generated.openapi.go"
+  - "**/*_test.go"
+  - "pkg/client/.*"
+  - "vendor/.*"
+  - "test/.*"
+  - "rust/**/error.rs"
+  - "rust/numaflow-models/**" # ignore generated files
+  - "rust/numaflow-pb/**" # ignore generated files
 coverage:
   status:
     patch: off

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.23"
       - name: Add bins to PATH
         run: |
           echo /home/runner/go/bin >> $GITHUB_PATH
@@ -81,7 +81,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.23"
         id: go
 
       - name: Check out code
@@ -145,7 +145,7 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.23"
       - name: Restore Go build cache
         uses: actions/cache@v4
         with:
@@ -233,7 +233,7 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v5
         with:
-          go-version: "1.22"
+          go-version: "1.23"
       - name: Add bins to PATH
         run: |
           echo /home/runner/go/bin >> $GITHUB_PATH

diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml
@@ -21,7 +21,7 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.23'
       - name: build
         run: make docs
       - name: deploy

diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml
@@ -26,7 +26,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.23'
 
       - name: Build binaries
         run: |

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -24,7 +24,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.23'
 
       - name: Build binaries
         run: |
@@ -162,7 +162,7 @@ jobs:
           fi
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: '1.23'
       - uses: actions/checkout@v4
       - run: go install sigs.k8s.io/bom/cmd/[email protected]
       - run: go install github.com/spdx/spdx-sbom-generator/cmd/[email protected]
@@ -240,4 +240,4 @@ jobs:
             /tmp/sbom.tar.gz
             /tmp/sbom.tar.gz.sig
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,20 @@
 # Changelog
 
+## v1.3.3 (2024-10-09)
+
+ * [4f31aad7](https://github.com/numaproj/numaflow/commit/4f31aad7f51cce59700ef53f363d06afeb6d6aee) Update manifests to v1.3.3
+ * [d0133636](https://github.com/numaproj/numaflow/commit/d01336364b1826c9d28ff81828919b17ca8da222) fix: pipeline pausing race conditions of draining and terminating source (#2131)
+ * [688dd730](https://github.com/numaproj/numaflow/commit/688dd73049617511806779a4e535ad9f380af21f) feat: expose ports for user defined containers (#2135)
+ * [a4a4fd05](https://github.com/numaproj/numaflow/commit/a4a4fd0578f7a4e45a6435505d03061c3612ed6f) fix: create buffers and buckets before updating Vertices (#2112)
+ * [498583f2](https://github.com/numaproj/numaflow/commit/498583f24573649f6ed2db959742f515804a2edc) chore(deps): bump rollup from 2.79.1 to 2.79.2 in /ui (#2096)
+
+### Contributors
+
+ * Derek Wang
+ * Julie Vogelman
+ * Sidhant Kohli
+ * dependabot[bot]
+
 ## v1.3.2 (2024-09-26)
 
  * [cb7d17d4](https://github.com/numaproj/numaflow/commit/cb7d17d4f3e2ecfcf6a1aa413031f714c135983d) Update manifests to v1.3.2

diff --git a/Dockerfile b/Dockerfile
@@ -18,12 +18,11 @@ RUN chmod +x /bin/numaflow-rs
 ####################################################################################################
 # Rust binary
 ####################################################################################################
-FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.81 AS chef
 ARG TARGETPLATFORM
 WORKDIR /numaflow
 RUN apt-get update && apt-get install -y protobuf-compiler
 
-
 FROM chef AS planner
 COPY ./rust/ .
 RUN cargo chef prepare --recipe-path recipe.json
@@ -90,4 +89,4 @@ RUN chmod +x /bin/e2eapi
 ####################################################################################################
 FROM scratch AS e2eapi
 COPY --from=testbase /bin/e2eapi .
-ENTRYPOINT ["/e2eapi"]
+ENTRYPOINT ["/e2eapi"]
diff --git a/Makefile b/Makefile
@@ -192,12 +192,22 @@ endif
 .PHONY: build-rust-in-docker
 build-rust-in-docker:
 	mkdir -p dist
-	-$(DOCKER) container ls --all --filter=ancestor='$(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)' --format "{{.ID}}" | xargs docker rm
+	-$(DOCKER) container ls --all --filter=ancestor='$(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)' --format "{{.ID}}" | xargs $(DOCKER) rm
 	-$(DOCKER) image rm $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)
 	DOCKER_BUILDKIT=1 $(DOCKER) build --build-arg "BASE_IMAGE=$(DEV_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION) --target rust-builder -f $(DOCKERFILE) .
-	export CTR=$$(docker create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)) && $(DOCKER) cp $$CTR:/root/numaflow dist/numaflow-rs-linux-$(HOST_ARCH) && $(DOCKER) rm $$CTR && $(DOCKER) image rm $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)
+	export CTR=$$($(DOCKER) create $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)) && $(DOCKER) cp $$CTR:/root/numaflow dist/numaflow-rs-linux-$(HOST_ARCH) && $(DOCKER) rm $$CTR && $(DOCKER) image rm $(IMAGE_NAMESPACE)/$(BINARY_NAME)-rust-builder:$(VERSION)
+
+.PHONY: build-rust-in-docker-multi
+build-rust-in-docker-multi:
+	mkdir -p dist
+	docker run -v ./dist/cargo:/root/.cargo -v ./rust/:/app/ -w /app --rm ubuntu:24.04 bash build.sh
+	cp -pv rust/target/aarch64-unknown-linux-gnu/release/numaflow dist/numaflow-rs-linux-arm64
+	cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow dist/numaflow-rs-linux-amd64
 
 image-multi: ui-build set-qemu dist/$(BINARY_NAME)-linux-arm64.gz dist/$(BINARY_NAME)-linux-amd64.gz
+ifndef GITHUB_ACTIONS
+	$(MAKE) build-rust-in-docker-multi
+endif
 	$(DOCKER) buildx build --sbom=false --provenance=false --build-arg "BASE_IMAGE=$(RELEASE_BASE_IMAGE)" $(DOCKER_BUILD_ARGS) -t $(IMAGE_NAMESPACE)/$(BINARY_NAME):$(VERSION) --target $(BINARY_NAME) --platform linux/amd64,linux/arm64 --file $(DOCKERFILE) ${PUSH_OPTION} .
 
 set-qemu:
@@ -213,6 +223,10 @@ swagger:
 api/json-schema/schema.json: api/openapi-spec/swagger.json hack/json-schema/main.go
 	go run ./hack/json-schema
 
+.PHONY: rustgen
+rustgen:
+	$(MAKE) --directory rust generate
+
 .PHONY: codegen
 codegen:
 	./hack/generate-proto.sh
@@ -223,7 +237,7 @@ codegen:
 	$(MAKE) manifests
 	rm -rf ./vendor
 	go mod tidy
-	$(MAKE) --directory rust/numaflow-models generate
+	$(MAKE) rustgen
 
 clean:
 	-rm -rf ${CURRENT_DIR}/dist

diff --git a/USERS.md b/USERS.md
@@ -8,3 +8,4 @@ Please add your company name and initial use case (optional) below.
 4. [Valegachain Analytics](https://www.valegachain.com/) Numaflow is used to extract, transform, and load cryptocurrency blocks and mempool transactions in data lakes, as well as for activity alerts.
 5. [Lockheed Martin](https://lockheedmartin.com/) Perform ELT processing on high and low volume data streams of sensor data as recieved from IOT type systems.
 6. [Seekr](https://www.seekr.com/) Numaflow coordinates multiple ML pipelines to rate and extract information from the pipeline input.
+7. [BEUMER Group](https://www.beumergroup.com/) - Signal processing for IoT-sensors, providing predictive maintenance solutions for the Intra-logistics sector.