From 5ef8182e27273cf62df176216eb7843a813e0c62 Mon Sep 17 00:00:00 2001 From: Gus Class Date: Mon, 7 Jan 2019 16:08:14 -0800 Subject: [PATCH 1/3] Increases reliability for flaky tests --- iot/api-client/manager/manager_test.py | 18 +++++++++++++--- iot/api-client/manager/requirements.txt | 2 ++ .../cloudiot_mqtt_example_test.py | 21 ++++++++++++++----- iot/api-client/mqtt_example/requirements.txt | 4 +++- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/iot/api-client/manager/manager_test.py b/iot/api-client/manager/manager_test.py index fb115dec118d..f7d12979e681 100644 --- a/iot/api-client/manager/manager_test.py +++ b/iot/api-client/manager/manager_test.py @@ -18,6 +18,7 @@ # Add command receiver for bootstrapping device registry / device for testing sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'mqtt_example')) # noqa +from gcp_devrel.testing.flaky import flaky from google.cloud import pubsub import pytest @@ -278,14 +279,25 @@ def test_add_patch_delete_es256(test_topic, capsys): service_account_json, project_id, cloud_region, registry_id) +@flaky def test_send_command(test_topic, capsys): device_id = device_id_template.format('RSA256') manager.create_registry( service_account_json, project_id, cloud_region, pubsub_topic, registry_id) - manager.create_rs256_device( - service_account_json, project_id, cloud_region, registry_id, - device_id, rsa_cert_path) + + + exists = False + devices = manager.list_devices( + service_account_json, project_id, cloud_region, registry_id) + for device in devices: + if device.get('id') == device_id: + exists = True + + if not exists: + manager.create_rs256_device( + service_account_json, project_id, cloud_region, registry_id, + device_id, rsa_cert_path) # Exercize the functionality client = cloudiot_mqtt_example.get_client( diff --git a/iot/api-client/manager/requirements.txt b/iot/api-client/manager/requirements.txt index 44105208ad27..d8aaefea5805 100644 --- a/iot/api-client/manager/requirements.txt +++ b/iot/api-client/manager/requirements.txt @@ -1,4 +1,6 @@ cryptography==2.4.2 +flaky==3.4.0 +gcp-devrel-py-tools==0.0.15 google-api-python-client==1.7.5 google-auth-httplib2==0.0.3 google-auth==1.6.1 diff --git a/iot/api-client/mqtt_example/cloudiot_mqtt_example_test.py b/iot/api-client/mqtt_example/cloudiot_mqtt_example_test.py index 47fb5d3e7f3e..a11f1f3efdf3 100644 --- a/iot/api-client/mqtt_example/cloudiot_mqtt_example_test.py +++ b/iot/api-client/mqtt_example/cloudiot_mqtt_example_test.py @@ -20,6 +20,7 @@ # Add manager for bootstrapping device registry / device for testing sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'manager')) # noqa +from gcp_devrel.testing.flaky import flaky import manager import pytest @@ -178,14 +179,24 @@ def test_config(test_topic, capsys): assert '/devices/{}/config'.format(device_id) in out +@flaky def test_receive_command(capsys): device_id = device_id_template.format('RSA256') manager.create_registry( service_account_json, project_id, cloud_region, pubsub_topic, registry_id) - manager.create_rs256_device( - service_account_json, project_id, cloud_region, registry_id, - device_id, rsa_cert_path) + + exists = False + devices = manager.list_devices( + service_account_json, project_id, cloud_region, registry_id) + for device in devices: + if device.get('id') == device_id: + exists = True + + if not exists: + manager.create_rs256_device( + service_account_json, project_id, cloud_region, registry_id, + device_id, rsa_cert_path) # Exercize the functionality client = cloudiot_mqtt_example.get_client( @@ -195,7 +206,7 @@ def test_receive_command(capsys): client.loop_start() # Pre-process commands - for i in range(1, 3): + for i in range(1, 5): client.loop() time.sleep(1) @@ -204,7 +215,7 @@ def test_receive_command(capsys): device_id, 'me want cookies') # Process commands - for i in range(1, 3): + for i in range(1, 5): client.loop() time.sleep(1) diff --git a/iot/api-client/mqtt_example/requirements.txt b/iot/api-client/mqtt_example/requirements.txt index 16beaa6e4212..15887683155c 100644 --- a/iot/api-client/mqtt_example/requirements.txt +++ b/iot/api-client/mqtt_example/requirements.txt @@ -1,7 +1,9 @@ +cryptography==2.4.2 +flaky==3.4.0 +gcp-devrel-py-tools==0.0.15 google-api-python-client==1.7.5 google-auth-httplib2==0.0.3 google-auth==1.6.1 google-cloud-pubsub==0.39.0 -cryptography==2.4.2 pyjwt==1.6.4 paho-mqtt==1.4.0 From 52a6a4d6bb3b91355438526180fabbe66fe3887f Mon Sep 17 00:00:00 2001 From: Gus Class Date: Mon, 7 Jan 2019 16:36:26 -0800 Subject: [PATCH 2/3] Fix for lint --- iot/api-client/manager/manager_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/iot/api-client/manager/manager_test.py b/iot/api-client/manager/manager_test.py index f7d12979e681..84f014960e40 100644 --- a/iot/api-client/manager/manager_test.py +++ b/iot/api-client/manager/manager_test.py @@ -286,7 +286,6 @@ def test_send_command(test_topic, capsys): service_account_json, project_id, cloud_region, pubsub_topic, registry_id) - exists = False devices = manager.list_devices( service_account_json, project_id, cloud_region, registry_id) From 2e5b001ab6b568b5a4e7dae5db829dbeb051fdcb Mon Sep 17 00:00:00 2001 From: Gus Class Date: Wed, 9 Jan 2019 22:39:19 -0800 Subject: [PATCH 3/3] Replaces checks where device or gateway ID could affect test outcome. --- iot/api-client/beta-features/gateway/gateway_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/iot/api-client/beta-features/gateway/gateway_test.py b/iot/api-client/beta-features/gateway/gateway_test.py index 50491e4c6fb6..fc039cc270b7 100644 --- a/iot/api-client/beta-features/gateway/gateway_test.py +++ b/iot/api-client/beta-features/gateway/gateway_test.py @@ -168,7 +168,6 @@ def test_create_gateway(iot_topic, capsys): out, _ = capsys.readouterr() assert 'Created gateway' in out - assert '400' not in out def test_list_gateways(iot_topic, capsys): @@ -195,7 +194,6 @@ def test_list_gateways(iot_topic, capsys): out, _ = capsys.readouterr() assert 'Gateway ID: {}'.format(gateway_id) in out - assert '400' not in out def test_bind_device_to_gateway_and_unbind(iot_topic, capsys): @@ -233,7 +231,7 @@ def test_bind_device_to_gateway_and_unbind(iot_topic, capsys): assert 'Device Bound' in out assert 'Device unbound' in out - assert '400' not in out + assert 'HttpError 404' not in out def test_gateway_listen_for_bound_device_configs(iot_topic, capsys):