diff --git a/gcloud/connection.py b/gcloud/connection.py index 5ede41322086..70c44d596acd 100644 --- a/gcloud/connection.py +++ b/gcloud/connection.py @@ -16,6 +16,7 @@ import json from pkg_resources import get_distribution +import six from six.moves.urllib.parse import urlencode # pylint: disable=F0401 import httplib2 @@ -295,6 +296,8 @@ def api_request(self, method, path, query_params=None, content_type = response.get('content-type', '') if not content_type.startswith('application/json'): raise TypeError('Expected JSON, got %s' % content_type) + if isinstance(content, six.binary_type): + content = content.decode('utf-8') return json.loads(content) return content diff --git a/gcloud/credentials.py b/gcloud/credentials.py index 37ce209e69b4..f4f6222be3ee 100644 --- a/gcloud/credentials.py +++ b/gcloud/credentials.py @@ -181,6 +181,8 @@ def _get_signed_query_params(credentials, expiration, signature_string): pem_key = _get_pem_key(credentials) # Sign the string with the RSA key. signer = PKCS1_v1_5.new(pem_key) + if not isinstance(signature_string, six.binary_type): + signature_string = signature_string.encode('utf-8') signature_hash = SHA256.new(signature_string) signature_bytes = signer.sign(signature_hash) signature = base64.b64encode(signature_bytes) diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py index c748d258968e..f91ae06b7482 100644 --- a/gcloud/datastore/test_connection.py +++ b/gcloud/datastore/test_connection.py @@ -152,7 +152,7 @@ def test__request_not_200(self): METHOD = 'METHOD' DATA = 'DATA' conn = self._makeOne() - conn._http = Http({'status': '400'}, 'Entity value is indexed.') + conn._http = Http({'status': '400'}, b'Entity value is indexed.') with self.assertRaises(BadRequest) as e: conn._request(DATASET_ID, METHOD, DATA) expected_message = '400 Entity value is indexed.' diff --git a/gcloud/exceptions.py b/gcloud/exceptions.py index 78f641118031..a2fe8ae6a3f5 100644 --- a/gcloud/exceptions.py +++ b/gcloud/exceptions.py @@ -18,6 +18,7 @@ """ import json +import six _HTTP_CODE_TO_EXCEPTION = {} # populated at end of module @@ -171,18 +172,18 @@ def make_exception(response, content, use_json=True): :rtype: instance of :class:`GCloudError`, or a concrete subclass. :returns: Exception specific to the error response. """ - message = content - errors = () + if isinstance(content, six.binary_type): + content = content.decode('utf-8') - if isinstance(content, str): + if isinstance(content, six.string_types): if use_json: payload = json.loads(content) else: - payload = {} + payload = {'message': content} else: payload = content - message = payload.get('message', message) + message = payload.get('message', '') errors = payload.get('error', {}).get('errors', ()) try: diff --git a/gcloud/storage/batch.py b/gcloud/storage/batch.py index 999065af5b7a..d7d4e3a71a28 100644 --- a/gcloud/storage/batch.py +++ b/gcloud/storage/batch.py @@ -170,10 +170,25 @@ def __exit__(self, exc_type, exc_val, exc_tb): def _unpack_batch_response(response, content): """Convert response, content -> [(status, reason, payload)].""" parser = Parser() - faux_message = ('Content-Type: %s\nMIME-Version: 1.0\n\n%s' % - (response['content-type'], content)) - - message = parser.parsestr(faux_message) + # We coerce to bytes to get consitent concat across + # Py2 and Py3. Percent formatting is insufficient since + # it includes the b in Py3. + if not isinstance(content, six.binary_type): + content = content.encode('utf-8') + content_type = response['content-type'] + if not isinstance(content_type, six.binary_type): + content_type = content_type.encode('utf-8') + faux_message = b''.join([ + b'Content-Type: ', + content_type, + b'\nMIME-Version: 1.0\n\n', + content, + ]) + + if six.PY2: + message = parser.parsestr(faux_message) + else: # pragma: NO COVER Python3 + message = parser.parsestr(faux_message.decode('utf-8')) if not isinstance(message._payload, list): raise ValueError('Bad response: not multi-part') diff --git a/gcloud/storage/test_api.py b/gcloud/storage/test_api.py index 44039b1e6f52..69bd5a2fee5e 100644 --- a/gcloud/storage/test_api.py +++ b/gcloud/storage/test_api.py @@ -34,7 +34,7 @@ def test_miss(self): ]) http = conn._http = Http( {'status': '404', 'content-type': 'application/json'}, - '{}', + b'{}', ) bucket = self._callFUT(NONESUCH, connection=conn) self.assertEqual(bucket, None) @@ -56,7 +56,7 @@ def _lookup_bucket_hit_helper(self, use_default=False): ]) http = conn._http = Http( {'status': '200', 'content-type': 'application/json'}, - '{"name": "%s"}' % BLOB_NAME, + '{{"name": "{0}"}}'.format(BLOB_NAME).encode('utf-8'), ) if use_default: @@ -96,7 +96,7 @@ def test_empty(self): ]) http = conn._http = Http( {'status': '200', 'content-type': 'application/json'}, - '{}', + b'{}', ) buckets = list(self._callFUT(PROJECT, conn)) self.assertEqual(len(buckets), 0) @@ -117,7 +117,8 @@ def _get_all_buckets_non_empty_helper(self, project, use_default=False): ]) http = conn._http = Http( {'status': '200', 'content-type': 'application/json'}, - '{"items": [{"name": "%s"}]}' % BUCKET_NAME, + '{{"items": [{{"name": "{0}"}}]}}'.format(BUCKET_NAME) + .encode('utf-8'), ) if use_default: @@ -159,7 +160,7 @@ def test_miss(self): ]) http = conn._http = Http( {'status': '404', 'content-type': 'application/json'}, - '{}', + b'{}', ) self.assertRaises(NotFound, self._callFUT, NONESUCH, connection=conn) self.assertEqual(http._called_with['method'], 'GET') @@ -180,7 +181,7 @@ def _get_bucket_hit_helper(self, use_default=False): ]) http = conn._http = Http( {'status': '200', 'content-type': 'application/json'}, - '{"name": "%s"}' % BLOB_NAME, + '{{"name": "{0}"}}'.format(BLOB_NAME).encode('utf-8'), ) if use_default: @@ -224,7 +225,7 @@ def _create_bucket_success_helper(self, project, use_default=False): ]) http = conn._http = Http( {'status': '200', 'content-type': 'application/json'}, - '{"name": "%s"}' % BLOB_NAME, + '{{"name": "{0}"}}'.format(BLOB_NAME).encode('utf-8'), ) if use_default: diff --git a/gcloud/storage/test_batch.py b/gcloud/storage/test_batch.py index 2f7569f45f83..aaa2e94dc10e 100644 --- a/gcloud/storage/test_batch.py +++ b/gcloud/storage/test_batch.py @@ -347,7 +347,32 @@ def test_as_context_mgr_w_error(self): self.assertEqual(len(batch._responses), 0) -_THREE_PART_MIME_RESPONSE = """\ +class Test__unpack_batch_response(unittest2.TestCase): + + def _callFUT(self, response, content): + from gcloud.storage.batch import _unpack_batch_response + return _unpack_batch_response(response, content) + + def test_bytes(self): + RESPONSE = {'content-type': b'multipart/mixed; boundary="DEADBEEF="'} + CONTENT = _THREE_PART_MIME_RESPONSE + result = list(self._callFUT(RESPONSE, CONTENT)) + self.assertEqual(len(result), 3) + self.assertEqual(result[0], ('200', 'OK', {u'bar': 2, u'foo': 1})) + self.assertEqual(result[1], ('200', 'OK', {u'foo': 1, u'bar': 3})) + self.assertEqual(result[2], ('204', 'No Content', '')) + + def test_unicode(self): + RESPONSE = {'content-type': u'multipart/mixed; boundary="DEADBEEF="'} + CONTENT = _THREE_PART_MIME_RESPONSE.decode('utf-8') + result = list(self._callFUT(RESPONSE, CONTENT)) + self.assertEqual(len(result), 3) + self.assertEqual(result[0], ('200', 'OK', {u'bar': 2, u'foo': 1})) + self.assertEqual(result[1], ('200', 'OK', {u'foo': 1, u'bar': 3})) + self.assertEqual(result[2], ('204', 'No Content', '')) + + +_THREE_PART_MIME_RESPONSE = b"""\ --DEADBEEF= Content-Type: application/http Content-ID: diff --git a/gcloud/test_connection.py b/gcloud/test_connection.py index 8b6261ba3725..06f70f3e8890 100644 --- a/gcloud/test_connection.py +++ b/gcloud/test_connection.py @@ -161,12 +161,12 @@ def test__make_request_no_data_no_content_type_no_headers(self): URI = 'http://example.com/test' http = conn._http = _Http( {'status': '200', 'content-type': 'text/plain'}, - '', + b'', ) headers, content = conn._make_request('GET', URI) self.assertEqual(headers['status'], '200') self.assertEqual(headers['content-type'], 'text/plain') - self.assertEqual(content, '') + self.assertEqual(content, b'') self.assertEqual(http._called_with['method'], 'GET') self.assertEqual(http._called_with['uri'], URI) self.assertEqual(http._called_with['body'], None) @@ -182,7 +182,7 @@ def test__make_request_w_data_no_extra_headers(self): URI = 'http://example.com/test' http = conn._http = _Http( {'status': '200', 'content-type': 'text/plain'}, - '', + b'', ) conn._make_request('GET', URI, {}, 'application/json') self.assertEqual(http._called_with['method'], 'GET') @@ -201,7 +201,7 @@ def test__make_request_w_extra_headers(self): URI = 'http://example.com/test' http = conn._http = _Http( {'status': '200', 'content-type': 'text/plain'}, - '', + b'', ) conn._make_request('GET', URI, headers={'X-Foo': 'foo'}) self.assertEqual(http._called_with['method'], 'GET') @@ -226,7 +226,7 @@ def test_api_request_defaults(self): ]) http = conn._http = _Http( {'status': '200', 'content-type': 'application/json'}, - '{}', + b'{}', ) self.assertEqual(conn.api_request('GET', PATH), {}) self.assertEqual(http._called_with['method'], 'GET') @@ -243,7 +243,7 @@ def test_api_request_w_non_json_response(self): conn = self._makeMockOne() conn._http = _Http( {'status': '200', 'content-type': 'text/plain'}, - 'CONTENT', + b'CONTENT', ) self.assertRaises(TypeError, conn.api_request, 'GET', '/') @@ -252,10 +252,10 @@ def test_api_request_wo_json_expected(self): conn = self._makeMockOne() conn._http = _Http( {'status': '200', 'content-type': 'text/plain'}, - 'CONTENT', + b'CONTENT', ) self.assertEqual(conn.api_request('GET', '/', expect_json=False), - 'CONTENT') + b'CONTENT') def test_api_request_w_query_params(self): from six.moves.urllib.parse import parse_qsl @@ -263,7 +263,7 @@ def test_api_request_w_query_params(self): conn = self._makeMockOne() http = conn._http = _Http( {'status': '200', 'content-type': 'application/json'}, - '{}', + b'{}', ) self.assertEqual(conn.api_request('GET', '/', {'foo': 'bar'}), {}) self.assertEqual(http._called_with['method'], 'GET') @@ -302,7 +302,7 @@ def test_api_request_w_data(self): ]) http = conn._http = _Http( {'status': '200', 'content-type': 'application/json'}, - '{}', + b'{}', ) self.assertEqual(conn.api_request('POST', '/', data=DATA), {}) self.assertEqual(http._called_with['method'], 'POST') @@ -321,7 +321,7 @@ def test_api_request_w_404(self): conn = self._makeMockOne() conn._http = _Http( {'status': '404', 'content-type': 'text/plain'}, - '{}' + b'{}' ) self.assertRaises(NotFound, conn.api_request, 'GET', '/') @@ -330,10 +330,35 @@ def test_api_request_w_500(self): conn = self._makeMockOne() conn._http = _Http( {'status': '500', 'content-type': 'text/plain'}, - '{}', + b'{}', ) self.assertRaises(InternalServerError, conn.api_request, 'GET', '/') + def test_api_request_non_binary_response(self): + conn = self._makeMockOne() + http = conn._http = _Http( + {'status': '200', 'content-type': 'application/json'}, + u'{}', + ) + result = conn.api_request('GET', '/') + # Intended to emulate self.mock_template + URI = '/'.join([ + conn.API_BASE_URL, + 'mock', + conn.API_VERSION, + '', + ]) + self.assertEqual(result, {}) + self.assertEqual(http._called_with['method'], 'GET') + self.assertEqual(http._called_with['uri'], URI) + self.assertEqual(http._called_with['body'], None) + expected_headers = { + 'Accept-Encoding': 'gzip', + 'Content-Length': 0, + 'User-Agent': conn.USER_AGENT, + } + self.assertEqual(http._called_with['headers'], expected_headers) + class _Http(object): diff --git a/gcloud/test_credentials.py b/gcloud/test_credentials.py index bb9c223c8408..6e743cd83269 100644 --- a/gcloud/test_credentials.py +++ b/gcloud/test_credentials.py @@ -176,11 +176,13 @@ def _get_pem_key(credentials): SIGNATURE_STRING = 'dummy_signature' with _Monkey(MUT, RSA=rsa, PKCS1_v1_5=pkcs_v1_5, SHA256=sha256, _get_pem_key=_get_pem_key): - self.assertRaises(NameError, self._callFUT, + self.assertRaises(UnboundLocalError, self._callFUT, BAD_CREDENTIALS, EXPIRATION, SIGNATURE_STRING) - def _run_test_with_credentials(self, credentials, account_name): + def _run_test_with_credentials(self, credentials, account_name, + signature_string=None): import base64 + import six from gcloud._testing import _Monkey from gcloud import credentials as MUT @@ -190,7 +192,7 @@ def _run_test_with_credentials(self, credentials, account_name): sha256 = _SHA256() EXPIRATION = '100' - SIGNATURE_STRING = b'dummy_signature' + SIGNATURE_STRING = signature_string or b'dummy_signature' with _Monkey(MUT, crypt=crypt, RSA=rsa, PKCS1_v1_5=pkcs_v1_5, SHA256=sha256): result = self._callFUT(credentials, EXPIRATION, SIGNATURE_STRING) @@ -199,7 +201,12 @@ def _run_test_with_credentials(self, credentials, account_name): self.assertEqual(crypt._private_key_text, base64.b64encode(b'dummy_private_key_text')) self.assertEqual(crypt._private_key_password, 'notasecret') - self.assertEqual(sha256._signature_string, SIGNATURE_STRING) + # sha256._signature_string is always bytes. + if isinstance(SIGNATURE_STRING, six.binary_type): + self.assertEqual(sha256._signature_string, SIGNATURE_STRING) + else: + self.assertEqual(sha256._signature_string, + SIGNATURE_STRING.encode('utf-8')) SIGNED = base64.b64encode(b'DEADBEEF') expected_query = { 'Expires': EXPIRATION, @@ -217,6 +224,17 @@ def test_signed_jwt_for_p12(self): ACCOUNT_NAME, b'dummy_private_key_text', scopes) self._run_test_with_credentials(credentials, ACCOUNT_NAME) + def test_signature_non_bytes(self): + from oauth2client import client + + scopes = [] + ACCOUNT_NAME = 'dummy_service_account_name' + SIGNATURE_STRING = u'dummy_signature' + credentials = client.SignedJwtAssertionCredentials( + ACCOUNT_NAME, b'dummy_private_key_text', scopes) + self._run_test_with_credentials(credentials, ACCOUNT_NAME, + signature_string=SIGNATURE_STRING) + def test_service_account_via_json_key(self): from oauth2client import service_account from gcloud._testing import _Monkey diff --git a/gcloud/test_exceptions.py b/gcloud/test_exceptions.py index ad7f89798660..b2d0ec2f7b0b 100644 --- a/gcloud/test_exceptions.py +++ b/gcloud/test_exceptions.py @@ -55,7 +55,7 @@ def _callFUT(self, response, content): def test_hit_w_content_as_str(self): from gcloud.exceptions import NotFound response = _Response(404) - content = '{"message": "Not Found"}' + content = b'{"message": "Not Found"}' exception = self._callFUT(response, content) self.assertTrue(isinstance(exception, NotFound)) self.assertEqual(exception.message, 'Not Found') diff --git a/regression/storage.py b/regression/storage.py index 544105668b77..9d9a7f585514 100644 --- a/regression/storage.py +++ b/regression/storage.py @@ -13,6 +13,7 @@ # limitations under the License. import httplib2 +import six import tempfile import time import unittest2 @@ -122,7 +123,10 @@ def test_large_file_write_from_stream(self): blob.upload_from_file(file_obj) self.case_blobs_to_delete.append(blob) - self.assertEqual(blob.md5_hash, file_data['hash']) + md5_hash = blob.md5_hash + if not isinstance(md5_hash, six.binary_type): + md5_hash = md5_hash.encode('utf-8') + self.assertEqual(md5_hash, file_data['hash']) def test_small_file_write_from_filename(self): blob = storage.Blob(bucket=self.bucket, name='SmallFile') @@ -132,7 +136,10 @@ def test_small_file_write_from_filename(self): blob.upload_from_filename(file_data['path']) self.case_blobs_to_delete.append(blob) - self.assertEqual(blob.md5_hash, file_data['hash']) + md5_hash = blob.md5_hash + if not isinstance(md5_hash, six.binary_type): + md5_hash = md5_hash.encode('utf-8') + self.assertEqual(md5_hash, file_data['hash']) def test_write_metadata(self): blob = self.bucket.upload_file(self.FILES['logo']['path']) @@ -145,14 +152,14 @@ def test_write_metadata(self): def test_direct_write_and_read_into_file(self): blob = storage.Blob(bucket=self.bucket, name='MyBuffer') - file_contents = 'Hello World' + file_contents = b'Hello World' blob.upload_from_string(file_contents) self.case_blobs_to_delete.append(blob) same_blob = storage.Blob(bucket=self.bucket, name='MyBuffer') same_blob._reload_properties() # Initialize properties. temp_filename = tempfile.mktemp() - with open(temp_filename, 'w') as file_obj: + with open(temp_filename, 'wb') as file_obj: same_blob.download_to_file(file_obj) with open(temp_filename, 'rb') as file_obj: @@ -299,7 +306,7 @@ def setUp(self): super(TestStorageSignURLs, self).setUp() logo_path = self.FILES['logo']['path'] - with open(logo_path, 'r') as file_obj: + with open(logo_path, 'rb') as file_obj: self.LOCAL_FILE = file_obj.read() blob = storage.Blob(bucket=self.bucket, name='LogoToSign.jpg') @@ -328,7 +335,7 @@ def test_create_signed_delete_url(self): response, content = HTTP.request(signed_delete_url, method='DELETE') self.assertEqual(response.status, 204) - self.assertEqual(content, '') + self.assertEqual(content, b'') # Check that the blob has actually been deleted. self.assertFalse(blob.name in self.bucket) diff --git a/tox.ini b/tox.ini index b4c53ad25234..03e669373a2f 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,8 @@ commands = {toxinidir}/scripts/run_regression.sh deps = unittest2 -# Use a development checkout of oauth2client until a release is made -# which fixes https://github.com/google/oauth2client/issues/125 - -egit+https://github.com/google/oauth2client.git#egg=oauth2client + # Use a development checkout of httplib2 until a release is made + # incorporating https://github.com/jcgregorio/httplib2/pull/291 + # and https://github.com/jcgregorio/httplib2/pull/296 + -egit+https://github.com/jcgregorio/httplib2.git#egg=httplib2 protobuf>=3.0.0-alpha-1