Fix flaky tests

docs/reference/limitations.md

@@ -44,4 +44,5 @@
 | Limitation | Motivation |
 | :--- | :--- |
 | Once data has been ingested into Feast, there is currently no way to delete the data without manually going to the database and deleting it. However, during retrieval only the latest rows will be returned for a specific key \(`event_timestamp`, `entity`\) based on its `created_timestamp`. | This functionality simply doesn't exist yet as a Feast API |
+| During the ingestion of data into BigQuery, `event_timestamp` is rounded down to seconds. E.g., `2020-08-21T08:40:19.906 -> 2020-08-21T08:40:19.000` | This ensures that floating point rounding errors do not occur during the retrieval of feature data, since this step requires time based joins |
 

infra/scripts/test-end-to-end-batch-dataflow.sh

@@ -15,6 +15,7 @@ test -z ${HELM_RELEASE_NAME} && HELM_RELEASE_NAME="pr-$PULL_NUMBER"
 test -z ${HELM_COMMON_NAME} && HELM_COMMON_NAME="deps"
 test -z ${DATASET_NAME} && DATASET_NAME=feast_e2e_$(date +%s)
 test -z ${SPECS_TOPIC} && SPECS_TOPIC=feast-specs-$(date +%s)
+test -z ${FEATURES_TOPIC} && FEATURES_TOPIC=feast-$(date +%s)
 
 
 feast_kafka_1_ip_name="feast-kafka-1"
@@ -212,6 +213,7 @@ export GCLOUD_REGION=$GCLOUD_REGION
 export HELM_COMMON_NAME=$HELM_COMMON_NAME
 export IMAGE_TAG=$PULL_PULL_SHA
 export SPECS_TOPIC=$SPECS_TOPIC
+export FEATURES_TOPIC=$FEATURES_TOPIC
 
 export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
 export SCRIPTS_DIR=${PROJECT_ROOT_DIR}/infra/scripts
@@ -220,7 +222,7 @@ source ${SCRIPTS_DIR}/setup-common-functions.sh
 wait_for_docker_image gcr.io/kf-feast/feast-core:"${IMAGE_TAG}"
 wait_for_docker_image gcr.io/kf-feast/feast-serving:"${IMAGE_TAG}"
 
-envsubst $'$TEMP_BUCKET $DATASET_NAME $GCLOUD_PROJECT $GCLOUD_NETWORK $SPECS_TOPIC \
+envsubst $'$TEMP_BUCKET $DATASET_NAME $GCLOUD_PROJECT $GCLOUD_NETWORK $SPECS_TOPIC $FEATURES_TOPIC \
   $GCLOUD_SUBNET $GCLOUD_REGION $IMAGE_TAG $HELM_COMMON_NAME $feast_kafka_1_ip
   $feast_kafka_2_ip $feast_kafka_3_ip $feast_redis_ip $feast_statsd_ip' < $ORIGINAL_DIR/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml > $ORIGINAL_DIR/infra/charts/feast/values-end-to-end-batch-dataflow-updated.yaml
 
@@ -288,7 +290,7 @@ jobcontroller_ip=$(kubectl get -o jsonpath="{.status.loadBalancer.ingress[0].ip}
 
 set +e
 pytest -s -v bq/bq-batch-retrieval.py -m dataflow_runner --core_url "$core_ip:6565" --serving_url "$serving_ip:6566" \
- --jobcontroller_url "$jobcontroller_ip:6570" --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+ --jobcontroller_url "$jobcontroller_ip:6570" --gcs_path "gs://${TEMP_BUCKET}" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
 TEST_EXIT_CODE=$?
 
 if [[ ${TEST_EXIT_CODE} != 0 ]]; then

infra/scripts/test-end-to-end-batch.sh

@@ -21,7 +21,7 @@ test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/et
 test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false"
 test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast"
 test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast"
-test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location"
+test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location/$(date +%s)"
 
 # Get the current build version using maven (and pom.xml)
 export FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
@@ -56,6 +56,12 @@ else
   echo "[DEBUG] Skipping building jars"
 fi
 
+DATASET_NAME=feast_$(date +%s)
+bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
+  --dataset \
+  --default_table_expiration 86400 \
+  ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
+
 # Start Feast Core in background
 cat <<EOF > /tmp/jc.warehouse.application.yml
 feast:
@@ -73,16 +79,6 @@ feast:
 
 EOF
 
-start_feast_core
-start_feast_jobcontroller /tmp/jc.warehouse.application.yml
-
-DATASET_NAME=feast_$(date +%s)
-bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
-  --dataset \
-  --default_table_expiration 86400 \
-  ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
-
-# Start Feast Online Serving in background
 cat <<EOF > /tmp/serving.warehouse.application.yml
 feast:
   # GRPC service address for Feast Core
@@ -122,6 +118,10 @@ server:
 
 EOF
 
+cat /tmp/jc.warehouse.application.yml /tmp/serving.warehouse.application.yml
+
+start_feast_core
+start_feast_jobcontroller /tmp/jc.warehouse.application.yml
 start_feast_serving /tmp/serving.warehouse.application.yml
 
 install_python_with_miniconda_and_feast_sdk
@@ -134,21 +134,20 @@ ORIGINAL_DIR=$(pwd)
 cd tests/e2e
 
 set +e
-pytest bq/* -m ${PYTEST_MARK} --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
+pytest bq/* -v -m ${PYTEST_MARK} --gcs_path ${JOBS_STAGING_LOCATION} --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
 TEST_EXIT_CODE=$?
 
 if [[ ${TEST_EXIT_CODE} != 0 ]]; then
   echo "[DEBUG] Printing logs"
   ls -ltrh /var/log/feast*
-  cat /var/log/feast-serving-warehouse.log /var/log/feast-core.log
+  cat /var/log/feast-serving-online.log /var/log/feast-core.log /var/log/feast-jobcontroller.log
 
   echo "[DEBUG] Printing Python packages list"
   pip list
-fi
-
-cd ${ORIGINAL_DIR}
+else
+  print_banner "Cleaning up"
 
-print_banner "Cleaning up"
+  bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
+fi
 
-bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
 exit ${TEST_EXIT_CODE}

infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml

@@ -15,6 +15,7 @@ feast-core:
       stream:
         options:
           bootstrapServers: $feast_kafka_1_ip:31090
+          topic: $FEATURES_TOPIC
 
 feast-jobcontroller:
   enabled: true

ingestion/pom.xml

@@ -144,6 +144,13 @@
       <version>${project.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>dev.feast</groupId>
+      <artifactId>feast-common-test</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>com.google.auto.value</groupId>
       <artifactId>auto-value-annotations</artifactId>
@@ -155,20 +162,6 @@
       <artifactId>google-cloud-bigquery</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>org.hamcrest</groupId>
-      <artifactId>hamcrest-core</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.hamcrest</groupId>
-      <artifactId>hamcrest-library</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-core</artifactId>
-    </dependency>
-
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
@@ -178,13 +171,6 @@
       <artifactId>protobuf-java-util</artifactId>
     </dependency>
 
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <version>4.12</version>
-      <scope>test</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
@@ -223,19 +209,6 @@
       <artifactId>slf4j-api</artifactId>
     </dependency>
 
-    <!-- To run actual Redis for ingestion integration test -->
-    <dependency>
-      <groupId>com.github.kstyrc</groupId>
-      <artifactId>embedded-redis</artifactId>
-    </dependency>
-
-    <!-- To run actual Kafka for ingestion integration test -->
-    <dependency>
-      <groupId>org.apache.kafka</groupId>
-      <artifactId>kafka_2.12</artifactId>
-      <scope>test</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-simple</artifactId>