diff --git a/tests/conformance/conftest.py b/tests/conformance/conftest.py new file mode 100644 index 000000000..91e9d0a71 --- /dev/null +++ b/tests/conformance/conftest.py @@ -0,0 +1,124 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid + +import pytest + +from google.auth.credentials import AnonymousCredentials +from google.cloud import storage +from google.cloud.exceptions import NotFound + + +"""Environment variable or default host for Storage testbench emulator.""" +_HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") + + +"""Emulated project information for the storage testbench.""" +_CONF_TEST_PROJECT_ID = "my-project-id" +_CONF_TEST_SERVICE_ACCOUNT_EMAIL = ( + "my-service-account@my-project-id.iam.gserviceaccount.com" +) +_CONF_TEST_PUBSUB_TOPIC_NAME = "my-topic-name" + +"""Create content payload in different sizes.""" + + +def _create_block(desired_kib): + line = "abc123XYZ" * 14 + "!" + "\n" + return 1024 * int(desired_kib / len(line)) * line + + +_STRING_CONTENT = "hello world" +_SIZE_16MB = 16384 # 16*1024 KiB + + +######################################################################################################################################## +### Pytest Fixtures to Populate Retry Conformance Test Resources ####################################################################### +######################################################################################################################################## + + +@pytest.fixture +def client(): + client = storage.Client( + project=_CONF_TEST_PROJECT_ID, + credentials=AnonymousCredentials(), + client_options={"api_endpoint": _HOST}, + ) + return client + + +@pytest.fixture +def bucket(client): + bucket = client.bucket(uuid.uuid4().hex) + client.create_bucket(bucket) + yield bucket + try: + bucket.delete(force=True) + except NotFound: # in cases where bucket is deleted within the test + pass + + +@pytest.fixture +def object(client, bucket): + blob = client.bucket(bucket.name).blob(uuid.uuid4().hex) + blob.upload_from_string(_STRING_CONTENT) + blob.reload() + yield blob + try: + blob.delete() + except NotFound: # in cases where object is deleted within the test + pass + + +@pytest.fixture +def notification(client, bucket): + notification = client.bucket(bucket.name).notification( + topic_name=_CONF_TEST_PUBSUB_TOPIC_NAME + ) + notification.create() + notification.reload() + yield notification + try: + notification.delete() + except NotFound: # in cases where notification is deleted within the test + pass + + +@pytest.fixture +def hmac_key(client): + hmac_key, _secret = client.create_hmac_key( + service_account_email=_CONF_TEST_SERVICE_ACCOUNT_EMAIL, + project_id=_CONF_TEST_PROJECT_ID, + ) + yield hmac_key + try: + hmac_key.state = "INACTIVE" + hmac_key.update() + hmac_key.delete() + except NotFound: # in cases where hmac_key is deleted within the test + pass + + +@pytest.fixture +def file_data(client, bucket): + blob = client.bucket(bucket.name).blob(uuid.uuid4().hex) + payload = _create_block(_SIZE_16MB) + blob.upload_from_string(payload) + yield blob, payload + try: + blob.delete() + except NotFound: # in cases where object is deleted within the test + pass diff --git a/tests/conformance/retry_strategy_test_data.json b/tests/conformance/retry_strategy_test_data.json index b807c6a72..9c2213560 100644 --- a/tests/conformance/retry_strategy_test_data.json +++ b/tests/conformance/retry_strategy_test_data.json @@ -239,6 +239,31 @@ ], "preconditionProvided": true, "expectSuccess": false + }, + { + "id": 7, + "description": "resumable_uploads_handle_complex_retries", + "cases": [], + "methods": [], + "preconditionProvided": true, + "expectSuccess": true + }, + { + "id": 8, + "description": "downloads_handle_complex_retries", + "cases": [ + { + "instructions": ["return-broken-stream", "return-broken-stream"] + }, + { + "instructions": ["return-broken-stream-after-256K"] + } + ], + "methods": [ + {"name": "storage.objects.get", "group": "storage.objects.download", "resources": ["BUCKET", "OBJECT"]} + ], + "preconditionProvided": false, + "expectSuccess": true } ] } \ No newline at end of file diff --git a/tests/conformance/test_conformance.py b/tests/conformance/test_conformance.py index cf4c026a8..ff70c6e16 100644 --- a/tests/conformance/test_conformance.py +++ b/tests/conformance/test_conformance.py @@ -24,12 +24,10 @@ import pytest import requests - -from six.moves.urllib import parse as urlparse +import urllib from google.auth.credentials import AnonymousCredentials from google.cloud import storage -from google.cloud.exceptions import NotFound from google.cloud.storage.hmac_key import HMACKeyMetadata from . import _read_local_json @@ -39,7 +37,7 @@ """Environment variable or default host for Storage testbench emulator.""" _HOST = os.environ.get("STORAGE_EMULATOR_HOST", "http://localhost:9000") -_PORT = urlparse.urlsplit(_HOST).port +_PORT = urllib.parse.urlsplit(_HOST).port """The storage testbench docker image info and commands.""" _DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" @@ -79,39 +77,72 @@ def blob_exists(client, _preconditions, **resources): def blob_download_as_bytes(client, _preconditions, **resources): bucket = resources.get("bucket") - object = resources.get("object") - blob = client.bucket(bucket.name).blob(object.name) - blob.download_as_bytes() + file, data = resources.get("file_data") + # download the file and assert data integrity + blob = client.bucket(bucket.name).blob(file.name) + stored_contents = blob.download_as_bytes() + assert stored_contents == data.encode("utf-8") + + +def blob_download_as_bytes_w_range(client, _preconditions, **resources): + bucket = resources.get("bucket") + file, data = resources.get("file_data") + blob = client.bucket(bucket.name).blob(file.name) + start_byte = 0 + end_byte = 1000000 + stored_contents = blob.download_as_bytes(start=start_byte, end=end_byte - 1) + assert stored_contents == data.encode("utf-8")[start_byte:end_byte] def blob_download_as_text(client, _preconditions, **resources): bucket = resources.get("bucket") - object = resources.get("object") - blob = client.bucket(bucket.name).blob(object.name) - blob.download_as_text() + file, data = resources.get("file_data") + blob = client.bucket(bucket.name).blob(file.name) + stored_contents = blob.download_as_text() + assert stored_contents == data def blob_download_to_filename(client, _preconditions, **resources): bucket = resources.get("bucket") - object = resources.get("object") - blob = client.bucket(bucket.name).blob(object.name) + file, data = resources.get("file_data") + blob = client.bucket(bucket.name).blob(file.name) + with tempfile.NamedTemporaryFile() as temp_f: + blob.download_to_filename(temp_f.name) + with open(temp_f.name, "r") as file_obj: + stored_contents = file_obj.read() + assert stored_contents == data + + +def blob_download_to_filename_chunked(client, _preconditions, **resources): + bucket = resources.get("bucket") + file, data = resources.get("file_data") + blob = client.bucket(bucket.name).blob(file.name, chunk_size=40 * 1024 * 1024) with tempfile.NamedTemporaryFile() as temp_f: blob.download_to_filename(temp_f.name) + with open(temp_f.name, "r") as file_obj: + stored_contents = file_obj.read() + assert stored_contents == data def client_download_blob_to_file(client, _preconditions, **resources): - object = resources.get("object") + bucket = resources.get("bucket") + file, data = resources.get("file_data") + blob = client.bucket(bucket.name).blob(file.name) with tempfile.NamedTemporaryFile() as temp_f: with open(temp_f.name, "wb") as file_obj: - client.download_blob_to_file(object, file_obj) + client.download_blob_to_file(blob, file_obj) + with open(temp_f.name, "r") as to_read: + stored_contents = to_read.read() + assert stored_contents == data def blobreader_read(client, _preconditions, **resources): bucket = resources.get("bucket") - object = resources.get("object") - blob = client.bucket(bucket.name).blob(object.name) - with blob.open() as reader: - reader.read() + file, data = resources.get("file_data") + blob = client.bucket(bucket.name).blob(file.name) + with blob.open(mode="r") as reader: + stored_contents = reader.read() + assert stored_contents == data def client_list_blobs(client, _preconditions, **resources): @@ -428,11 +459,14 @@ def blob_compose(client, _preconditions, **resources): def blob_upload_from_string(client, _preconditions, **resources): bucket = resources.get("bucket") + _, data = resources.get("file_data") blob = client.bucket(bucket.name).blob(uuid.uuid4().hex) + blob.chunk_size = 4 * 1024 * 1024 if _preconditions: - blob.upload_from_string(_STRING_CONTENT, if_generation_match=0) + blob.upload_from_string(data, if_generation_match=0) else: - blob.upload_from_string(_STRING_CONTENT) + blob.upload_from_string(data) + assert blob.size == len(data) def blob_upload_from_file(client, _preconditions, **resources): @@ -672,10 +706,20 @@ def object_acl_clear(client, _preconditions, **resources): blob_exists, client_download_blob_to_file, blob_download_to_filename, + blob_download_to_filename_chunked, blob_download_as_bytes, blob_download_as_text, blobreader_read, ], + "storage.objects.download": [ + client_download_blob_to_file, + blob_download_to_filename, + blob_download_to_filename_chunked, + blob_download_as_bytes, + blob_download_as_bytes_w_range, + blob_download_as_text, + blobreader_read, + ], "storage.objects.list": [client_list_blobs, bucket_list_blobs, bucket_delete], "storage.serviceaccount.get": [client_get_service_account_email], # S1 end "storage.buckets.patch": [ @@ -722,73 +766,6 @@ def object_acl_clear(client, _preconditions, **resources): } -######################################################################################################################################## -### Pytest Fixtures to Populate Resources ############################################################################################## -######################################################################################################################################## - - -@pytest.fixture -def client(): - client = storage.Client( - project=_CONF_TEST_PROJECT_ID, - credentials=AnonymousCredentials(), - client_options={"api_endpoint": _HOST}, - ) - return client - - -@pytest.fixture -def bucket(client): - bucket = client.bucket(uuid.uuid4().hex) - client.create_bucket(bucket) - yield bucket - try: - bucket.delete(force=True) - except NotFound: # in cases where bucket is deleted within the test - pass - - -@pytest.fixture -def object(client, bucket): - blob = client.bucket(bucket.name).blob(uuid.uuid4().hex) - blob.upload_from_string(_STRING_CONTENT) - blob.reload() - yield blob - try: - blob.delete() - except NotFound: # in cases where object is deleted within the test - pass - - -@pytest.fixture -def notification(client, bucket): - notification = client.bucket(bucket.name).notification( - topic_name=_CONF_TEST_PUBSUB_TOPIC_NAME - ) - notification.create() - notification.reload() - yield notification - try: - notification.delete() - except NotFound: # in cases where notification is deleted within the test - pass - - -@pytest.fixture -def hmac_key(client): - hmac_key, _secret = client.create_hmac_key( - service_account_email=_CONF_TEST_SERVICE_ACCOUNT_EMAIL, - project_id=_CONF_TEST_PROJECT_ID, - ) - yield hmac_key - try: - hmac_key.state = "INACTIVE" - hmac_key.update() - hmac_key.delete() - except NotFound: # in cases where hmac_key is deleted within the test - pass - - ######################################################################################################################################## ### Helper Methods for Testbench Retry Test API ######################################################################################## ######################################################################################################################################## @@ -833,7 +810,15 @@ def _get_retry_test(host, id): def _run_retry_test( - host, id, lib_func, _preconditions, bucket, object, notification, hmac_key + host, + id, + lib_func, + _preconditions, + bucket, + object, + notification, + hmac_key, + file_data, ): """ To execute tests against the list of instrucions sent to the Retry Test API, @@ -855,6 +840,7 @@ def _run_retry_test( object=object, notification=notification, hmac_key=hmac_key, + file_data=file_data, ) @@ -874,7 +860,16 @@ def _delete_retry_test(host, id): def run_test_case( - scenario_id, method, case, lib_func, host, bucket, object, notification, hmac_key + scenario_id, + method, + case, + lib_func, + host, + bucket, + object, + notification, + hmac_key, + file_data, ): scenario = _CONFORMANCE_TESTS[scenario_id - 1] expect_success = scenario["expectSuccess"] @@ -901,6 +896,7 @@ def run_test_case( object, notification, hmac_key, + file_data, ) except Exception as e: logging.exception( @@ -948,11 +944,12 @@ def run_test_case( for i, c in enumerate(cases): for m in methods: method_name = m["name"] - if method_name not in method_mapping: + method_group = m["group"] if m.get("group", None) else m["name"] + if method_group not in method_mapping: logging.info("No tests for operation {}".format(method_name)) continue - for lib_func in method_mapping[method_name]: + for lib_func in method_mapping[method_group]: test_name = "test-S{}-{}-{}-{}".format( id, method_name, lib_func.__name__, i )