Backport CI housekeeping to 0.6 (#923)

* Backport housekeeping of Feast CI, docker compose, and notebooks (#916) * Add caching to core and serving builds * Fix flaky e2e tests * Simplify docker-compose setup * Add kafka broker to docker compose test * Rename batch to historical * Update troubleshooting.md * Fix casing for FEAST_HISTORICAL_SERVING_URL * Remove f-string prefix * Fix missing key in docker compose setup * Fix documentation typos * Add jupyter to load test * Add docker compose back to load-test * Clean up FEAST_VERSION in load test * Fix artifact output for load test * Revert e2e test order * Add missing makefile commands from backport * Remove missing tests
feast-dev · Aug 2, 2020 · 9728b74 · 9728b74
1 parent 9f2f5da
commit 9728b74
Show file tree

Hide file tree

Showing 29 changed files with 530 additions and 283 deletions.
diff --git a/.github/workflows/code_standards.yaml b/.github/workflows/code_standards.yaml
diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml
@@ -0,0 +1,111 @@
+name: complete
+
+on: [push, pull_request]
+
+jobs:
+  build-push-docker-images:
+    runs-on: [self-hosted]
+    strategy:
+      matrix:
+        component: [core, serving, jupyter]
+    env:
+      GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }}
+      REGISTRY: gcr.io/kf-feast
+      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar
+    steps:
+      - uses: actions/checkout@v2
+      - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '290.0.1'
+          export_default_credentials: true
+      - run: gcloud auth configure-docker --quiet
+      - name: Get m2 cache
+        run: |
+          infra/scripts/download-maven-cache.sh \
+          --archive-uri ${MAVEN_CACHE} \
+          --output-dir .
+      - name: Build image
+        run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA}
+      - name: Push image
+        run: |
+          docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA}
+          if [ -n "${GITHUB_PR_SHA}" ]; then
+            docker tag ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_PR_SHA}
+            docker push ${REGISTRY}/feast-${{ matrix.component }}:${GITHUB_PR_SHA}
+          fi
+
+  lint-java:
+    container: gcr.io/kf-feast/feast-ci:latest
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Lint java
+        run: make lint-java
+
+  lint-python:
+    container: gcr.io/kf-feast/feast-ci:latest
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: make install-python-ci-dependencies
+      - name: Compile protos
+        run: make compile-protos-python
+      - name: Lint python
+        run: make lint-python
+
+  lint-go:
+    container: gcr.io/kf-feast/feast-ci:latest
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: make install-go-ci-dependencies
+      - name: Lint go
+        run: make lint-go
+
+  lint-versions:
+    container: gcr.io/kf-feast/feast-ci:latest
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - name: install dependencies
+        run: make lint-versions
+
+  unit-test-java:
+    runs-on: ubuntu-latest
+    container: gcr.io/kf-feast/feast-ci:latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v1
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Test java
+        run: make test-java-with-coverage
+      - uses: actions/upload-artifact@v2
+        with:
+          name: java-coverage-report
+          path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/
+
+  unit-test-python:
+    runs-on: ubuntu-latest
+    container: gcr.io/kf-feast/feast-ci:latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install python
+        run: make install-python
+      - name: Test python
+        run: make test-python
+
+  unit-test-go:
+    runs-on: ubuntu-latest
+    container: gcr.io/kf-feast/feast-ci:latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: make compile-protos-go
+      - name: Test go
+        run: make test-go
diff --git a/.github/workflows/docker_compose_tests.yml b/.github/workflows/docker_compose_tests.yml
diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml
@@ -0,0 +1,60 @@
+name: master only
+
+on:
+  push:
+    branches: master
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  build-docker-images:
+    runs-on: [self-hosted]
+    strategy:
+      matrix:
+        component: [core, serving, jupyter, ci]
+    env:
+      MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2019-10-24.tar
+    steps:
+      - uses: actions/checkout@v2
+      - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+        with:
+          version: '290.0.1'
+          export_default_credentials: true
+      - run: gcloud auth configure-docker --quiet
+      - name: Get m2 cache
+        run: |
+          infra/scripts/download-maven-cache.sh \
+          --archive-uri ${MAVEN_CACHE} \
+          --output-dir .
+      - name: Build image
+        run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
+      - name: Push image
+        run: make push-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA}
+      - name: Push image to feast dev
+        run: |
+          if [ ${GITHUB_REF#refs/*/} == "master" ]; then
+            docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:dev
+            docker push gcr.io/kf-feast/feast-${{ matrix.component }}:dev
+          fi
+      - name: Get version
+        run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/}
+      - name: Push versioned release
+        run: |
+          # Build and push semver tagged commits
+          rx='^v[0-9]+?\.[0-9]+?\.[0-9]+?$'
+          if [[ "${RELEASE_VERSION}" =~ $rx ]]; then
+            VERSION_WITHOUT_PREFIX=${RELEASE_VERSION:1}
+
+            docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
+            docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX}
+
+            # Also update "latest" image if tagged commit is pushed to stable branch
+            HIGHEST_SEMVER_TAG=$(git tag -l --sort -version:refname | head -n 1)
+            echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG"
+
+            if [ "${VERSION_WITHOUT_PREFIX}" == "${HIGHEST_SEMVER_TAG:1}" ]
+            then
+              docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:latest
+              docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest
+            fi
+          fi
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
diff --git a/Makefile b/Makefile
@@ -126,6 +126,9 @@ push-serving-docker:
 push-ci-docker:
 	docker push $(REGISTRY)/feast-ci:latest
 
+push-jupyter-docker:
+	docker push $(REGISTRY)/feast-jupyter:$(VERSION)
+
 build-core-docker:
 	docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile .
 
@@ -135,6 +138,9 @@ build-serving-docker:
 build-ci-docker:
 	docker build -t $(REGISTRY)/feast-ci:latest -f infra/docker/ci/Dockerfile .
 
+build-jupyter-docker:
+	docker build -t $(REGISTRY)/feast-jupyter:$(VERSION) -f infra/docker/jupyter/Dockerfile .
+
 # Documentation
 
 install-dependencies-proto-docs:
@@ -175,4 +181,7 @@ build-html: clean-html
 
 # Versions
 lint-versions:
-	./infra/scripts/validate-version-consistency.sh
+	./infra/scripts/validate-version-consistency.sh
+
+test-load:
+	./infra/scripts/test-load.sh $(GIT_SHA)
diff --git a/docs/administration/troubleshooting.md b/docs/administration/troubleshooting.md
@@ -15,8 +15,8 @@ docker ps
 ```text
 CONTAINER ID        IMAGE                                  COMMAND                  CREATED             STATUS              PORTS                                            NAMES
 d7447205bced        jupyter/datascience-notebook:latest    "tini -g -- start-no…"   2 minutes ago       Up 2 minutes        0.0.0.0:8888->8888/tcp                           feast_jupyter_1
-8e49dbe81b92        gcr.io/kf-feast/feast-serving:latest   "java -Xms1024m -Xmx…"   2 minutes ago       Up 5 seconds        0.0.0.0:6567->6567/tcp                           feast_batch-serving_1
-b859494bd33a        gcr.io/kf-feast/feast-serving:latest   "java -jar /opt/feas…"   2 minutes ago       Up About a minute   0.0.0.0:6566->6566/tcp                           feast_online-serving_1
+8e49dbe81b92        gcr.io/kf-feast/feast-serving:latest   "java -Xms1024m -Xmx…"   2 minutes ago       Up 5 seconds        0.0.0.0:6567->6567/tcp                           feast_historical_serving_1
+b859494bd33a        gcr.io/kf-feast/feast-serving:latest   "java -jar /opt/feas…"   2 minutes ago       Up About a minute   0.0.0.0:6566->6566/tcp                           feast_online_serving_1
 5c4962811767        gcr.io/kf-feast/feast-core:latest      "java -jar /opt/feas…"   2 minutes ago       Up 2 minutes        0.0.0.0:6565->6565/tcp                           feast_core_1
 1ba7239e0ae0        confluentinc/cp-kafka:5.2.1            "/etc/confluent/dock…"   2 minutes ago       Up 2 minutes        0.0.0.0:9092->9092/tcp, 0.0.0.0:9094->9094/tcp   feast_kafka_1
 e2779672735c        confluentinc/cp-zookeeper:5.2.1        "/etc/confluent/dock…"   2 minutes ago       Up 2 minutes        2181/tcp, 2888/tcp, 3888/tcp                     feast_zookeeper_1
@@ -59,8 +59,8 @@ You will probably need to connect using the hostnames of services and standard F
 
 ```bash
 export FEAST_CORE_URL=core:6565
-export FEAST_ONLINE_SERVING_URL=online-serving:6566
-export FEAST_BATCH_SERVING_URL=batch-serving:6567
+export FEAST_ONLINE_SERVING_URL=online_serving:6566
+export FEAST_HISTORICAL_SERVING_URL=historical_serving:6567
 ```
 
 ### **Docker Compose \(from outside the docker cluster\)**
@@ -70,7 +70,7 @@ You will probably need to connect using `localhost` and standard ports:
 ```bash
 export FEAST_CORE_URL=localhost:6565
 export FEAST_ONLINE_SERVING_URL=localhost:6566
-export FEAST_BATCH_SERVING_URL=localhost:6567
+export FEAST_HISTORICAL_SERVING_URL=localhost:6567
 ```
 
 ### **Google Kubernetes Engine \(GKE\)**
@@ -81,7 +81,7 @@ You will need to find the external IP of one of the nodes as well as the NodePor
 export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1)
 export FEAST_CORE_URL=${FEAST_IP}:32090
 export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091
-export FEAST_BATCH_SERVING_URL=${FEAST_IP}:32092
+export FEAST_HISTORICAL_SERVING_URL=${FEAST_IP}:32092
 ```
 
 `netcat`, `telnet`, or even `curl` can be used to test whether all services are available and ports are open, but `grpc_cli` is the most powerful. It can be installed from [here](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md).
@@ -107,7 +107,7 @@ ListProjects
 ### Testing Feast Batch Serving and Online Serving
 
 ```bash
-grpc_cli ls ${FEAST_BATCH_SERVING_URL} feast.serving.ServingService
+grpc_cli ls ${FEAST_HISTORICAL_SERVING_URL} feast.serving.ServingService
 ```
 
 ```text
@@ -145,11 +145,11 @@ In order to print the logs from these services, please run the commands below.
 ```
 
 ```text
-docker logs -f feast_batch-serving_1
+docker logs -f feast_historical_serving_1
 ```
 
 ```text
-docker logs -f feast_online-serving_1
+docker logs -f feast_online_serving_1
 ```
 
 ### Google Kubernetes Engine

diff --git a/docs/getting-started/deploying-feast/docker-compose.md b/docs/getting-started/deploying-feast/docker-compose.md
@@ -80,12 +80,12 @@ Configure the `.env` file based on your environment. At the very least you have
 | Parameter | Description |
 | :--- | :--- |
 | `FEAST_CORE_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json`. |
-| `FEAST_BATCH_SERVING_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json` |
+| `FEAST_HISTORICAL_SERVING_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json` |
 | `FEAST_JUPYTER_GCP_SERVICE_ACCOUNT_KEY` | This should be your service account file name, for example `key.json` |
 
 ### 3.3 Configure Historical Serving
 
-We will also need to configure the `batch-serving.yml` file inside `infra/docker-compose/serving/`. This configuration is used to retrieve training datasets from Feast. At a minimum you will need to set:
+We will also need to configure the `historical-serving.yml` file inside `infra/docker-compose/serving/`. This configuration is used to retrieve training datasets from Feast. At a minimum you will need to set:
 
 | Parameter | Description |
 | :--- | :--- |

diff --git a/docs/installation/docker-compose.md b/docs/installation/docker-compose.md
@@ -85,7 +85,7 @@ Configure the `.env` file based on your environment. At the very least you have
 | Parameter | Description |
 | :--- | :--- |
 | FEAST\_CORE\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json`. |
-| FEAST\_BATCH\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
+| FEAST\_HISTORICAL\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
 | FEAST\_JUPYTER\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` |
 | FEAST\_JOB\_STAGING\_LOCATION | Google Cloud Storage bucket that Feast will use to stage data exports and batch retrieval requests, for example `gs://your-gcs-bucket/staging` |