Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix flaky tests #953

Merged
merged 16 commits into from
Aug 25, 2020
Merged

Fix flaky tests #953

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
e46bc4c
debug flaky tests
pyalex Aug 21, 2020
7377808
trailing slash
pyalex Aug 21, 2020
6b72e3e
trailing slash
pyalex Aug 21, 2020
f97781b
debug
pyalex Aug 21, 2020
732db05
correct log path
pyalex Aug 21, 2020
f0f3300
testcontainers for ingestion unit tests
pyalex Aug 21, 2020
2c8f087
fix rounding error
pyalex Aug 21, 2020
e6a8451
cleanup if success
pyalex Aug 24, 2020
5b528d4
close resources
pyalex Aug 24, 2020
3e29338
unique feast topic per e2e test
pyalex Aug 24, 2020
26bbd8c
remove debug
pyalex Aug 24, 2020
ec0048f
f-string in python unit tests
pyalex Aug 25, 2020
1deb30a
lint
pyalex Aug 25, 2020
0411ced
round event timestamp to seconds
pyalex Aug 25, 2020
7121a6c
add line about event_timestamp to limitation
pyalex Aug 25, 2020
0ccf41b
add line about event_timestamp to limitation
pyalex Aug 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions infra/scripts/test-end-to-end-batch-dataflow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ test -z ${HELM_RELEASE_NAME} && HELM_RELEASE_NAME="pr-$PULL_NUMBER"
test -z ${HELM_COMMON_NAME} && HELM_COMMON_NAME="deps"
test -z ${DATASET_NAME} && DATASET_NAME=feast_e2e_$(date +%s)
test -z ${SPECS_TOPIC} && SPECS_TOPIC=feast-specs-$(date +%s)
test -z ${FEATURES_TOPIC} && FEATURES_TOPIC=feast-$(date +%s)


feast_kafka_1_ip_name="feast-kafka-1"
Expand Down Expand Up @@ -212,6 +213,7 @@ export GCLOUD_REGION=$GCLOUD_REGION
export HELM_COMMON_NAME=$HELM_COMMON_NAME
export IMAGE_TAG=$PULL_PULL_SHA
export SPECS_TOPIC=$SPECS_TOPIC
export FEATURES_TOPIC=$FEATURES_TOPIC

export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
export SCRIPTS_DIR=${PROJECT_ROOT_DIR}/infra/scripts
Expand All @@ -220,7 +222,7 @@ source ${SCRIPTS_DIR}/setup-common-functions.sh
wait_for_docker_image gcr.io/kf-feast/feast-core:"${IMAGE_TAG}"
wait_for_docker_image gcr.io/kf-feast/feast-serving:"${IMAGE_TAG}"

envsubst $'$TEMP_BUCKET $DATASET_NAME $GCLOUD_PROJECT $GCLOUD_NETWORK $SPECS_TOPIC \
envsubst $'$TEMP_BUCKET $DATASET_NAME $GCLOUD_PROJECT $GCLOUD_NETWORK $SPECS_TOPIC $FEATURES_TOPIC \
$GCLOUD_SUBNET $GCLOUD_REGION $IMAGE_TAG $HELM_COMMON_NAME $feast_kafka_1_ip
$feast_kafka_2_ip $feast_kafka_3_ip $feast_redis_ip $feast_statsd_ip' < $ORIGINAL_DIR/infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml > $ORIGINAL_DIR/infra/charts/feast/values-end-to-end-batch-dataflow-updated.yaml

Expand Down Expand Up @@ -288,7 +290,7 @@ jobcontroller_ip=$(kubectl get -o jsonpath="{.status.loadBalancer.ingress[0].ip}

set +e
pytest -s -v bq/bq-batch-retrieval.py -m dataflow_runner --core_url "$core_ip:6565" --serving_url "$serving_ip:6566" \
--jobcontroller_url "$jobcontroller_ip:6570" --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
--jobcontroller_url "$jobcontroller_ip:6570" --gcs_path "gs://${TEMP_BUCKET}" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
TEST_EXIT_CODE=$?

if [[ ${TEST_EXIT_CODE} != 0 ]]; then
Expand Down
35 changes: 17 additions & 18 deletions infra/scripts/test-end-to-end-batch.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ test -z ${GOOGLE_APPLICATION_CREDENTIALS} && GOOGLE_APPLICATION_CREDENTIALS="/et
test -z ${SKIP_BUILD_JARS} && SKIP_BUILD_JARS="false"
test -z ${GOOGLE_CLOUD_PROJECT} && GOOGLE_CLOUD_PROJECT="kf-feast"
test -z ${TEMP_BUCKET} && TEMP_BUCKET="feast-templocation-kf-feast"
test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location"
test -z ${JOBS_STAGING_LOCATION} && JOBS_STAGING_LOCATION="gs://${TEMP_BUCKET}/staging-location/$(date +%s)"

# Get the current build version using maven (and pom.xml)
export FEAST_BUILD_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
Expand Down Expand Up @@ -56,6 +56,12 @@ else
echo "[DEBUG] Skipping building jars"
fi

DATASET_NAME=feast_$(date +%s)
bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
--dataset \
--default_table_expiration 86400 \
${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}

# Start Feast Core in background
cat <<EOF > /tmp/jc.warehouse.application.yml
feast:
Expand All @@ -73,16 +79,6 @@ feast:

EOF

start_feast_core
start_feast_jobcontroller /tmp/jc.warehouse.application.yml

DATASET_NAME=feast_$(date +%s)
bq --location=US --project_id=${GOOGLE_CLOUD_PROJECT} mk \
--dataset \
--default_table_expiration 86400 \
${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}

# Start Feast Online Serving in background
cat <<EOF > /tmp/serving.warehouse.application.yml
feast:
# GRPC service address for Feast Core
Expand Down Expand Up @@ -122,6 +118,10 @@ server:

EOF

cat /tmp/jc.warehouse.application.yml /tmp/serving.warehouse.application.yml

start_feast_core
start_feast_jobcontroller /tmp/jc.warehouse.application.yml
start_feast_serving /tmp/serving.warehouse.application.yml

install_python_with_miniconda_and_feast_sdk
Expand All @@ -134,21 +134,20 @@ ORIGINAL_DIR=$(pwd)
cd tests/e2e

set +e
pytest bq/* -m ${PYTEST_MARK} --gcs_path "gs://${TEMP_BUCKET}/" --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
pytest bq/* -v -m ${PYTEST_MARK} --gcs_path ${JOBS_STAGING_LOCATION} --junitxml=${LOGS_ARTIFACT_PATH}/python-sdk-test-report.xml
TEST_EXIT_CODE=$?

if [[ ${TEST_EXIT_CODE} != 0 ]]; then
echo "[DEBUG] Printing logs"
ls -ltrh /var/log/feast*
cat /var/log/feast-serving-warehouse.log /var/log/feast-core.log
cat /var/log/feast-serving-online.log /var/log/feast-core.log /var/log/feast-jobcontroller.log

echo "[DEBUG] Printing Python packages list"
pip list
fi

cd ${ORIGINAL_DIR}
else
print_banner "Cleaning up"

print_banner "Cleaning up"
bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
fi

bq rm -r -f ${GOOGLE_CLOUD_PROJECT}:${DATASET_NAME}
exit ${TEST_EXIT_CODE}
1 change: 1 addition & 0 deletions infra/scripts/test-templates/values-end-to-end-batch-dataflow.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ feast-core:
stream:
options:
bootstrapServers: $feast_kafka_1_ip:31090
topic: $FEATURES_TOPIC

feast-jobcontroller:
enabled: true
Expand Down
41 changes: 7 additions & 34 deletions ingestion/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,13 @@
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>dev.feast</groupId>
<artifactId>feast-common-test</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value-annotations</artifactId>
Expand All @@ -155,20 +162,6 @@
<artifactId>google-cloud-bigquery</artifactId>
</dependency>

<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
</dependency>

<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</dependency>

<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
Expand All @@ -178,13 +171,6 @@
<artifactId>protobuf-java-util</artifactId>
</dependency>

<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
Expand Down Expand Up @@ -223,19 +209,6 @@
<artifactId>slf4j-api</artifactId>
</dependency>

<!-- To run actual Redis for ingestion integration test -->
<dependency>
<groupId>com.github.kstyrc</groupId>
<artifactId>embedded-redis</artifactId>
</dependency>

<!-- To run actual Kafka for ingestion integration test -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.12</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
Expand Down
Loading