From e16dba62f07a3dbffab1aeb0ac8c088bd8b1e144 Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 28 Apr 2018 21:48:12 +0800 Subject: [PATCH 1/9] cos python sdk support py3! --- README.rst | 2 +- qcloud_cos/cos_auth.py | 46 ++-- qcloud_cos/cos_client.py | 445 +++++++++++++++++++---------------- qcloud_cos/cos_comm.py | 198 +++++++++------- qcloud_cos/cos_threadpool.py | 6 +- qcloud_cos/demo.py | 24 +- qcloud_cos/xml2dict.py | 4 +- requirements.txt | 1 + ut/test.py | 29 +-- 9 files changed, 415 insertions(+), 340 deletions(-) diff --git a/README.rst b/README.rst index 172a61ce..66357a0a 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ Qcloud COSv5 SDK 介绍 _______ -腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7。 +腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7以及Python3.x。 安装指南 __________ diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 4a010443..887ed614 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- +from six.moves.urllib.parse import quote, unquote, urlparse, urlencode import hmac import time -import urllib import hashlib import logging -from urllib import quote -from urlparse import urlparse from requests.auth import AuthBase +from .cos_comm import to_unicode, to_bytes logger = logging.getLogger(__name__) @@ -18,61 +17,52 @@ def filter_headers(data): :return(dict): 计算进签名的头部. """ headers = {} - for i in data.keys(): + for i in data: if i == 'Content-Type' or i == 'Host' or i[0] == 'x' or i[0] == 'X': headers[i] = data[i] return headers -def to_string(data): - """转换unicode为string. - - :param data(unicode|string): 待转换的unicode|string. - :return(string): 转换后的string. - """ - if isinstance(data, unicode): - return data.encode('utf8') - return data - - class CosS3Auth(AuthBase): - def __init__(self, secret_id, secret_key, key='', params={}, expire=10000): - self._secret_id = to_string(secret_id) - self._secret_key = to_string(secret_key) + def __init__(self, secret_id, secret_key, key=None, params={}, expire=10000): + self._secret_id = secret_id + self._secret_key = secret_key self._expire = expire self._params = params if key: - if key[0] == '/': + key = to_unicode(key) + if key[0] == u'/': self._path = key else: - self._path = '/' + key + self._path = u'/' + key else: - self._path = '/' + self._path = u'/' def __call__(self, r): path = self._path uri_params = self._params headers = filter_headers(r.headers) # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) - headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode - format_str = "{method}\n{host}\n{params}\n{headers}\n".format( + headers = dict([(k.lower(), quote(to_bytes(v), '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode + uri_params = dict([(k.lower(), v) for k, v in uri_params.items()]) + format_str = u"{method}\n{host}\n{params}\n{headers}\n".format( method=r.method.lower(), host=path, - params=urllib.urlencode(sorted(uri_params.items())), - headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) + params=urlencode(sorted(uri_params.items())), + headers='&'.join(map(lambda tupl: "%s=%s" % (tupl[0], tupl[1]), sorted(headers.items()))) ) logger.debug("format str: " + format_str) start_sign_time = int(time.time()) sign_time = "{bg_time};{ed_time}".format(bg_time=start_sign_time-60, ed_time=start_sign_time+self._expire) sha1 = hashlib.sha1() - sha1.update(format_str) + sha1.update(to_bytes(format_str)) str_to_sign = "sha1\n{time}\n{sha1}\n".format(time=sign_time, sha1=sha1.hexdigest()) logger.debug('str_to_sign: ' + str(str_to_sign)) - sign_key = hmac.new(self._secret_key, sign_time, hashlib.sha1).hexdigest() - sign = hmac.new(sign_key, str_to_sign, hashlib.sha1).hexdigest() + sign_key = hmac.new(to_bytes(self._secret_key), to_bytes(sign_time), hashlib.sha1).hexdigest() + sign = hmac.new(to_bytes(sign_key), to_bytes(str_to_sign), hashlib.sha1).hexdigest() logger.debug('sign_key: ' + str(sign_key)) logger.debug('sign: ' + str(sign)) sign_tpl = "q-sign-algorithm=sha1&q-ak={ak}&q-sign-time={sign_time}&q-key-time={key_time}&q-header-list={headers}&q-url-param-list={params}&q-signature={sign}" diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 45e41c07..413ff74d 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -1,7 +1,6 @@ # -*- coding=utf-8 import requests -import urllib import logging import hashlib import base64 @@ -12,21 +11,18 @@ import xml.etree.ElementTree from requests import Request, Session from datetime import datetime -from urllib import quote +from six.moves.urllib.parse import quote, unquote from hashlib import md5 -from streambody import StreamBody -from xml2dict import Xml2Dict from dicttoxml import dicttoxml -from cos_auth import CosS3Auth -from cos_comm import * -from cos_threadpool import SimpleThreadPool -from cos_exception import CosClientError -from cos_exception import CosServiceError +from .streambody import StreamBody +from .xml2dict import Xml2Dict +from .cos_auth import CosS3Auth +from .cos_comm import * +from .cos_threadpool import SimpleThreadPool +from .cos_exception import CosClientError +from .cos_exception import CosServiceError logger = logging.getLogger(__name__) -reload(sys) -sys.setdefaultencoding('utf-8') - class CosConfig(object): """config类，保存用户相关信息""" @@ -43,30 +39,28 @@ def __init__(self, Appid=None, Region=None, Secret_id=None, Secret_key=None, Tok :param Access_id(string): 秘钥AccessId(兼容). :param Access_key(string): 秘钥AccessKey(兼容). """ - self._appid = Appid + self._appid = to_unicode(Appid) self._region = format_region(Region) - self._token = Token + self._token = to_unicode(Token) self._timeout = Timeout if Scheme is None: - Scheme = 'http' - if(Scheme != 'http' and Scheme != 'https'): + Scheme = u'http' + Scheme = to_unicode(Scheme) + if(Scheme != u'http' and Scheme != u'https'): raise CosClientError('Scheme can be only set to http/https') self._scheme = Scheme # 兼容(SecretId,SecretKey)以及(AccessId,AccessKey) if(Secret_id and Secret_key): - self._secret_id = Secret_id - self._secret_key = Secret_key + self._secret_id = to_unicode(Secret_id) + self._secret_key = to_unicode(Secret_key) elif(Access_id and Access_key): - self._secret_id = Access_id - self._secret_key = Access_key + self._secret_id = to_unicode(Access_id) + self._secret_key = to_unicode(Access_key) else: raise CosClientError('SecretId and SecretKey is Required!') - logger.info("config parameter-> appid: {appid}, region: {region}".format( - appid=Appid, - region=Region)) def uri(self, bucket, path=None, scheme=None, region=None): """拼接url @@ -81,14 +75,16 @@ def uri(self, bucket, path=None, scheme=None, region=None): if region is None: region = self._region if path is not None: - if path == "": - raise CosClientError("Key can't be empty string") - if path[0] == '/': + if not path: + raise CosClientError("Key is required not empty") + path = to_unicode(path) + if path[0] == u'/': path = path[1:] + path = quote(to_bytes(path), '/-_.~') url = u"{scheme}://{bucket}.{region}.myqcloud.com/{path}".format( - scheme=scheme, + scheme=to_unicode(scheme), bucket=to_unicode(bucket), - region=region, + region=to_unicode(region), path=to_unicode(path) ) else: @@ -140,9 +136,9 @@ def get_auth(self, Method, Bucket, Key, Expired=300, Headers={}, Params={}): Headers={'header1': 'value1'}, Params={'param1': 'value1'} ) - print auth_string + print (auth_string) """ - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) + url = self._conf.uri(bucket=Bucket, path=Key) r = Request(Method, url, headers=Headers, params=Params) auth = CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, Params, Expired) return auth(r).headers['Authorization'] @@ -154,8 +150,11 @@ def send_request(self, method, url, timeout=30, **kwargs): if self._conf._token is not None: kwargs['headers']['x-cos-security-token'] = self._conf._token kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.1' - try: - for j in range(self._retry): + kwargs['headers'] = format_values(kwargs['headers']) + if 'data' in kwargs: + kwargs['data'] = to_bytes(kwargs['data']) + #try: + for j in range(self._retry): if method == 'POST': res = self._session.post(url, timeout=timeout, **kwargs) elif method == 'GET': @@ -168,9 +167,9 @@ def send_request(self, method, url, timeout=30, **kwargs): res = self._session.head(url, timeout=timeout, **kwargs) if res.status_code < 400: # 2xx和3xx都认为是成功的 return res - except Exception as e: # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误 - logger.exception('url:%s, exception:%s' % (url, str(e))) - raise CosClientError(str(e)) + #except Exception as e: # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误 + #logger.exception('url:%s, exception:%s' % (url, str(e))) + #raise CosClientError(str(e)) if res.status_code >= 400: # 所有的4XX,5XX都认为是COSServiceError if method == 'HEAD' and res.status_code == 404: # Head 需要处理 @@ -184,7 +183,7 @@ def send_request(self, method, url, timeout=30, **kwargs): raise CosServiceError(method, info, res.status_code) else: msg = res.text - if msg == '': # 服务器没有返回Error Body时 给出头部的信息 + if msg == u'': # 服务器没有返回Error Body时 给出头部的信息 msg = res.headers logger.error(msg) raise CosServiceError(method, msg, res.status_code) @@ -213,16 +212,11 @@ def put_object(self, Bucket, Body, Key, EnableMD5=False, **kwargs): Body=fp, Key='test.txt' ) - print response['ETag'] + print (response['ETag']) """ check_object_content_length(Body) headers = mapped(kwargs) - if 'Metadata' in headers.keys(): - for i in headers['Metadata'].keys(): - headers[i] = headers['Metadata'][i] - headers.pop('Metadata') - - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) # 提前对key做encode + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -261,16 +255,23 @@ def get_object(self, Bucket, Key, **kwargs): response['Body'].get_stream_to_file('local_file.txt') """ headers = mapped(kwargs) + final_headers = {} params = {} - for key in headers.keys(): + for key in headers: if key.startswith("response"): params[key] = headers[key] - headers.pop(key) - if 'versionId' in headers.keys(): + else: + final_headers[key] = headers[key] + + headers = final_headers + + if 'versionId' in headers: params['versionId'] = headers['versionId'] - headers.pop('versionId') + del headers['versionId'] + + params = format_values(params) - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("get object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -278,7 +279,7 @@ def get_object(self, Bucket, Key, **kwargs): method='GET', url=url, stream=True, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), params=params, headers=headers) @@ -305,9 +306,9 @@ def get_presigned_download_url(self, Bucket, Key, Expired=300): Key='test.txt' ) """ - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) + url = self._conf.uri(bucket=Bucket, path=Key) sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key, Expired=300) - url = url + '?sign=' + urllib.quote(sign) + url = url + '?sign=' + quote(sign) return url def delete_object(self, Bucket, Key, **kwargs): @@ -330,10 +331,10 @@ def delete_object(self, Bucket, Key, **kwargs): """ headers = mapped(kwargs) params = {} - if 'versionId' in headers.keys(): + if 'versionId' in headers: params['versionId'] = headers['versionId'] - headers.pop('versionId') - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) + del headers['versionId'] + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("delete object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -380,17 +381,20 @@ def delete_objects(self, Bucket, Delete={}, **kwargs): headers = mapped(kwargs) headers['Content-MD5'] = get_md5(xml_config) headers['Content-Type'] = 'application/xml' - url = self._conf.uri(bucket=Bucket, path="?delete") - logger.info("put bucket replication, url=:{url} ,headers=:{headers}".format( + params = {'delete': ''} + params = format_values(params) + url = self._conf.uri(bucket=Bucket) + logger.info("delete objects, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - data = xml_to_dict(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content) format_dict(data, ['Deleted', 'Error']) return data @@ -414,17 +418,17 @@ def head_object(self, Bucket, Key, **kwargs): """ headers = mapped(kwargs) params = {} - if 'versionId' in headers.keys(): + if 'versionId' in headers: params['versionId'] = headers['versionId'] - headers.pop('versionId') - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) + del headers['versionId'] + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("head object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='HEAD', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), headers=headers, params=params) return rt.headers @@ -452,15 +456,11 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): ) """ headers = mapped(kwargs) - if 'Metadata' in headers.keys(): - for i in headers['Metadata'].keys(): - headers[i] = headers['Metadata'][i] - headers.pop('Metadata') headers['x-cos-copy-source'] = gen_copy_source_url(CopySource) if CopyStatus != 'Copy' and CopyStatus != 'Replaced': raise CosClientError('CopyStatus must be Copy or Replaced') headers['x-cos-metadata-directive'] = CopyStatus - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("copy object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -469,7 +469,7 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): url=url, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), headers=headers) - body = xml_to_dict(rt.text) + body = xml_to_dict(rt.content) data = rt.headers data.update(body) return data @@ -503,9 +503,9 @@ def upload_part_copy(self, Bucket, Key, PartNumber, UploadId, CopySource, CopySo headers = mapped(kwargs) headers['x-cos-copy-source'] = gen_copy_source_url(CopySource) headers['x-cos-copy-source-range'] = CopySourceRange - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?partNumber={PartNumber}&uploadId={UploadId}".format( - PartNumber=PartNumber, - UploadId=UploadId)) + params = {'partNumber': PartNumber, 'uploadId': UploadId} + params = format_values(params) + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("upload part copy, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -513,8 +513,9 @@ def upload_part_copy(self, Bucket, Key, PartNumber, UploadId, CopySource, CopySo method='PUT', url=url, headers=headers, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key)) - body = xml_to_dict(rt.text) + params=params, + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params)) + body = xml_to_dict(rt.content) data = rt.headers data.update(body) return data @@ -538,22 +539,20 @@ def create_multipart_upload(self, Bucket, Key, **kwargs): ) """ headers = mapped(kwargs) - if 'Metadata' in headers.keys(): - for i in headers['Metadata'].keys(): - headers[i] = headers['Metadata'][i] - headers.pop('Metadata') - - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploads") + params = {'uploads': ''} + params = format_values(params) + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), + headers=headers, + params=params) - data = xml_to_dict(rt.text) + data = xml_to_dict(rt.content) return data def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, EnableMD5=False, **kwargs): @@ -583,10 +582,10 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, EnableMD5=False, """ check_object_content_length(Body) headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?partNumber={PartNumber}&uploadId={UploadId}".format( - PartNumber=PartNumber, - UploadId=UploadId)) - logger.info("put object, url=:{url} ,headers=:{headers}".format( + params = {'partNumber': PartNumber, 'uploadId': UploadId} + params = format_values(params) + url = self._conf.uri(bucket=Bucket, path=Key) + logger.info("upload part, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) Body = deal_with_empty_file_stream(Body) @@ -598,7 +597,8 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, EnableMD5=False, method='PUT', url=url, headers=headers, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), + params=params, + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), data=Body) response = dict() response['ETag'] = rt.headers['ETag'] @@ -627,18 +627,21 @@ def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, * ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) - logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format( + params = {'uploadId': UploadId} + params = format_values(params) + url = self._conf.uri(bucket=Bucket, path=Key) + logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), data=dict_to_xml(MultipartUpload), timeout=1200, # 分片上传大文件的时间比较长，设置为20min - headers=headers) - body = xml_to_dict(rt.text) + headers=headers, + params=params) + body = xml_to_dict(rt.content) data = rt.headers data.update(body) return data @@ -664,15 +667,18 @@ def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) + params = {'uploadId': UploadId} + params = format_values(params) + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), + headers=headers, + params=params) return None def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, PartNumberMarker=0, **kwargs): @@ -711,18 +717,19 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, Part decodeflag = False else: params['encoding-type'] = 'url' - - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) - logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format( + + params = format_values(params) + url = self._conf.uri(bucket=Bucket, path=Key) + logger.info("list multipart upload parts, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), headers=headers, params=params) - data = xml_to_dict(rt.text) + data = xml_to_dict(rt.content) format_dict(data, ['Part']) if decodeflag: decode_result(data, ['Key'], []) @@ -756,7 +763,8 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): if AccessControlPolicy: xml_config = format_xml(data=AccessControlPolicy, root='AccessControlPolicy', lst=lst) headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl") + params = {'acl': ''} + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("put object acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -764,8 +772,9 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), + headers=headers, + params=params) return None def get_object_acl(self, Bucket, Key, **kwargs): @@ -787,16 +796,18 @@ def get_object_acl(self, Bucket, Key, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl") + params = {'acl': ''} + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("get object acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), - headers=headers) - data = xml_to_dict(rt.text, "type", "Type") + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content, "type", "Type") if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict): lst = [] lst.append(data['AccessControlList']['Grant']) @@ -812,9 +823,9 @@ def restore_object(self, Bucket, Key, RestoreRequest={}, **kwargs): :param kwargs(dict): 设置请求headers. :return: None. """ - params = {} + params = {'restore': ''} headers = mapped(kwargs) - if 'versionId' in headers.keys(): + if 'versionId' in headers: params['versionId'] = headers['versionId'] headers.pop('versionId') url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?restore") @@ -826,7 +837,7 @@ def restore_object(self, Bucket, Key, RestoreRequest={}, **kwargs): method='POST', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), headers=headers, params=params) return None @@ -931,13 +942,14 @@ def list_objects(self, Bucket, Prefix="", Delimiter="", Marker="", MaxKeys=1000, params['encoding-type'] = EncodingType else: params['encoding-type'] = 'url' + params = format_values(params) rt = self.send_request( method='GET', url=url, params=params, headers=headers, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) - data = xml_to_dict(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params)) + data = xml_to_dict(rt.content) format_dict(data, ['Contents', 'CommonPrefixes']) if decodeflag: decode_result( @@ -981,11 +993,12 @@ def list_objects_versions(self, Bucket, Prefix="", Delimiter="", KeyMarker="", V """ headers = mapped(kwargs) decodeflag = True - url = self._conf.uri(bucket=Bucket, path='?versions') + url = self._conf.uri(bucket=Bucket) logger.info("list objects versions, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) params = { + 'versions': '', 'prefix': Prefix, 'delimiter': Delimiter, 'key-marker': KeyMarker, @@ -999,13 +1012,14 @@ def list_objects_versions(self, Bucket, Prefix="", Delimiter="", KeyMarker="", V params['encoding-type'] = EncodingType else: params['encoding-type'] = 'url' + params = format_values(params) rt = self.send_request( method='GET', url=url, params=params, headers=headers, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) - data = xml_to_dict(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params)) + data = xml_to_dict(rt.content) format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes']) if decodeflag: decode_result( @@ -1052,11 +1066,12 @@ def list_multipart_uploads(self, Bucket, Prefix="", Delimiter="", KeyMarker="", """ headers = mapped(kwargs) decodeflag = True - url = self._conf.uri(bucket=Bucket, path='?uploads') + url = self._conf.uri(bucket=Bucket) logger.info("get multipart uploads, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) params = { + 'uploads': '', 'prefix': Prefix, 'delimiter': Delimiter, 'key-marker': KeyMarker, @@ -1070,14 +1085,15 @@ def list_multipart_uploads(self, Bucket, Prefix="", Delimiter="", KeyMarker="", params['encoding-type'] = EncodingType else: params['encoding-type'] = 'url' + params = format_values(params) rt = self.send_request( method='GET', url=url, params=params, headers=headers, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key)) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params)) - data = xml_to_dict(rt.text) + data = xml_to_dict(rt.content) format_dict(data, ['Upload', 'CommonPrefixes']) if decodeflag: decode_result( @@ -1150,7 +1166,8 @@ def put_bucket_acl(self, Bucket, AccessControlPolicy={}, **kwargs): if AccessControlPolicy: xml_config = format_xml(data=AccessControlPolicy, root='AccessControlPolicy', lst=lst) headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?acl") + params = {'acl': ''} + url = self._conf.uri(bucket=Bucket) logger.info("put bucket acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -1158,8 +1175,9 @@ def put_bucket_acl(self, Bucket, AccessControlPolicy={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def get_bucket_acl(self, Bucket, **kwargs): @@ -1179,16 +1197,18 @@ def get_bucket_acl(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?acl") + params = {'acl': ''} + url = self._conf.uri(bucket=Bucket) logger.info("get bucket acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - data = xml_to_dict(rt.text, "type", "Type") + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content, "type", "Type") if data['AccessControlList'] is not None and not isinstance(data['AccessControlList']['Grant'], list): lst = [] lst.append(data['AccessControlList']['Grant']) @@ -1240,7 +1260,8 @@ def put_bucket_cors(self, Bucket, CORSConfiguration={}, **kwargs): headers = mapped(kwargs) headers['Content-MD5'] = get_md5(xml_config) headers['Content-Type'] = 'application/xml' - url = self._conf.uri(bucket=Bucket, path="?cors") + params = {'cors': ''} + url = self._conf.uri(bucket=Bucket) logger.info("put bucket cors, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -1248,8 +1269,9 @@ def put_bucket_cors(self, Bucket, CORSConfiguration={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def get_bucket_cors(self, Bucket, **kwargs): @@ -1269,25 +1291,27 @@ def get_bucket_cors(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?cors") + params = {'cors': ''} + url = self._conf.uri(bucket=Bucket) logger.info("get bucket cors, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - data = xml_to_dict(rt.text) - if 'CORSRule' in data.keys() and not isinstance(data['CORSRule'], list): + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content) + if 'CORSRule' in data and not isinstance(data['CORSRule'], list): lst = [] lst.append(data['CORSRule']) data['CORSRule'] = lst - if 'CORSRule' in data.keys(): + if 'CORSRule' in data: allow_lst = ['AllowedOrigin', 'AllowedMethod', 'AllowedHeader', 'ExposeHeader'] for rule in data['CORSRule']: for text in allow_lst: - if text in rule.keys() and not isinstance(rule[text], list): + if text in rule and not isinstance(rule[text], list): lst = [] lst.append(rule[text]) rule[text] = lst @@ -1310,15 +1334,17 @@ def delete_bucket_cors(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?cors") + params = {'cors': ''} + url = self._conf.uri(bucket=Bucket) logger.info("delete bucket cors, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs): @@ -1364,7 +1390,8 @@ def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs): headers = mapped(kwargs) headers['Content-MD5'] = get_md5(xml_config) headers['Content-Type'] = 'application/xml' - url = self._conf.uri(bucket=Bucket, path="?lifecycle") + params = {'lifecycle': ''} + url = self._conf.uri(bucket=Bucket) logger.info("put bucket lifecycle, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -1372,8 +1399,9 @@ def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def get_bucket_lifecycle(self, Bucket, **kwargs): @@ -1393,21 +1421,23 @@ def get_bucket_lifecycle(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?lifecycle") + params = {'lifecycle': ''} + url = self._conf.uri(bucket=Bucket) logger.info("get bucket lifecycle, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - data = xml_to_dict(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content) format_dict(data, ['Rule']) - if 'Rule' in data.keys(): + if 'Rule' in data: for rule in data['Rule']: format_dict(rule, ['Transition', 'NoncurrentVersionTransition']) - if 'Filter' in rule.keys(): + if 'Filter' in rule: format_dict(rule['Filter'], ['Tag']) return data @@ -1428,15 +1458,17 @@ def delete_bucket_lifecycle(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?lifecycle") - logger.info("delete bucket cors, url=:{url} ,headers=:{headers}".format( + params = {'lifecycle': ''} + url = self._conf.uri(bucket=Bucket) + logger.info("delete bucket lifecycle, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def put_bucket_versioning(self, Bucket, Status, **kwargs): @@ -1458,7 +1490,8 @@ def put_bucket_versioning(self, Bucket, Status, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?versioning") + params = {'versioning': ''} + url = self._conf.uri(bucket=Bucket) logger.info("put bucket versioning, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -1471,8 +1504,9 @@ def put_bucket_versioning(self, Bucket, Status, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def get_bucket_versioning(self, Bucket, **kwargs): @@ -1492,16 +1526,18 @@ def get_bucket_versioning(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?versioning") + params = {'versioning': ''} + url = self._conf.uri(bucket=Bucket) logger.info("get bucket versioning, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - data = xml_to_dict(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content) return data def get_bucket_location(self, Bucket, **kwargs): @@ -1519,19 +1555,21 @@ def get_bucket_location(self, Bucket, **kwargs): response = client.get_bucket_location( Bucket='bucket' ) - print response['LocationConstraint'] + print (response['LocationConstraint']) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?location") + params = {'location': ''} + url = self._conf.uri(bucket=Bucket) logger.info("get bucket location, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - root = xml.etree.ElementTree.fromstring(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + root = xml.etree.ElementTree.fromstring(rt.content) data = dict() data['LocationConstraint'] = root.text return data @@ -1572,7 +1610,8 @@ def put_bucket_replication(self, Bucket, ReplicationConfiguration={}, **kwargs): headers = mapped(kwargs) headers['Content-MD5'] = get_md5(xml_config) headers['Content-Type'] = 'application/xml' - url = self._conf.uri(bucket=Bucket, path="?replication") + params = {'replication': ''} + url = self._conf.uri(bucket=Bucket) logger.info("put bucket replication, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -1580,8 +1619,9 @@ def put_bucket_replication(self, Bucket, ReplicationConfiguration={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def get_bucket_replication(self, Bucket, **kwargs): @@ -1601,16 +1641,18 @@ def get_bucket_replication(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?replication") + params = {'replication': ''} + url = self._conf.uri(bucket=Bucket) logger.info("get bucket replication, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - data = xml_to_dict(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content) format_dict(data, ['Rule']) return data @@ -1631,15 +1673,17 @@ def delete_bucket_replication(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?replication") + params = {'replication': ''} + url = self._conf.uri(bucket=Bucket) logger.info("delete bucket replication, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) return None def put_bucket_logging(self, Bucket, BucketLoggingStatus={}, **kwargs): @@ -1671,7 +1715,8 @@ def put_bucket_logging(self, Bucket, BucketLoggingStatus={}, **kwargs): headers = mapped(kwargs) headers['Content-MD5'] = get_md5(xml_config) headers['Content-Type'] = 'application/xml' - url = self._conf.uri(bucket=Bucket, path="?logging") + params = {'logging': ''} + url = self._conf.uri(bucket=Bucket) logger.info("put bucket logging, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -1679,8 +1724,9 @@ def put_bucket_logging(self, Bucket, BucketLoggingStatus={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) grant_rt = self.put_bucket_acl(Bucket=Bucket, GrantFullControl=LOGGING_UIN) return None @@ -1701,16 +1747,18 @@ def get_bucket_logging(self, Bucket, **kwargs): ) """ headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path="?logging") + params = {'logging': ''} + url = self._conf.uri(bucket=Bucket) logger.info("get bucket logging, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), - headers=headers) - data = xml_to_dict(rt.text) + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, params=params), + headers=headers, + params=params) + data = xml_to_dict(rt.content) return data # service interface begin @@ -1736,7 +1784,7 @@ def list_buckets(self, **kwargs): headers=headers, auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key), ) - data = xml_to_dict(rt.text) + data = xml_to_dict(rt.content) if data['Buckets'] is not None and not isinstance(data['Buckets']['Bucket'], list): lst = [] lst.append(data['Buckets']['Bucket']) @@ -1781,7 +1829,7 @@ def _get_resumable_uploadid(self, bucket, key): Bucket=bucket, Prefix=key ) - if 'Upload' in multipart_response.keys(): + if 'Upload' in multipart_response: if multipart_response['Upload'][0]['Key'] == key: return multipart_response['Upload'][0]['UploadId'] @@ -1884,14 +1932,14 @@ def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, **kwa else: part_size = 1024*1024*PartSize # 默认按照1MB分块,最大支持10G的文件，超过10G的分块数固定为10000 last_size = 0 # 最后一块可以小于1MB - parts_num = file_size / part_size + parts_num = file_size // part_size last_size = file_size % part_size if last_size != 0: parts_num += 1 if parts_num > 10000: parts_num = 10000 - part_size = file_size / parts_num + part_size = file_size // parts_num last_size = file_size % parts_num last_size += part_size @@ -1936,11 +1984,11 @@ def _inner_head_object(self, CopySource): params = {} if versionid != '': params['versionId'] = versionid - url = self._conf.uri(bucket=bucket, path=quote(path, '/-_.~'), scheme=self._conf._scheme, region=region) + url = self._conf.uri(bucket=bucket, path=path, scheme=self._conf._scheme, region=region) rt = self.send_request( method='HEAD', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, path), + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, path, params=params), headers={}, params=params) return int(rt.headers['Content-Length']) @@ -1962,7 +2010,7 @@ def _upload_part_copy(self, bucket, key, part_number, upload_id, copy_source, co return None def _check_same_region(self, dst_region, CopySource): - if 'Region' in CopySource.keys(): + if 'Region' in CopySource: src_region = CopySource['Region'] src_region = format_region(src_region) else: @@ -2011,13 +2059,13 @@ def copy(self, Bucket, Key, CopySource, CopyStatus='Copy', PartSize=10, MAXThrea # 如果源文件大小大于等于5G，则先创建分块上传，在调用upload_part part_size = 1024*1024*PartSize # 默认按照10MB分块 last_size = 0 # 最后一块可以小于1MB - parts_num = file_size / part_size + parts_num = file_size // part_size last_size = file_size % part_size if last_size != 0: parts_num += 1 if parts_num > 10000: parts_num = 10000 - part_size = file_size / parts_num + part_size = file_size // parts_num last_size = file_size % parts_num last_size += part_size # 创建分块上传 @@ -2098,7 +2146,7 @@ def upload_file_from_buffer(self, Bucket, Key, Body, MaxBufferSize=100, PartSize uploadid = rt['UploadId'] lst = list() # 记录分块信息 - MAXQueue = MaxBufferSize/PartSize + MAXQueue = MaxBufferSize//PartSize pool = SimpleThreadPool(MAXThread, MAXQueue) while True: if data == "": @@ -2132,12 +2180,8 @@ def append_object(self, Bucket, Key, Position, Data, **kwargs): :return(dict): 上传成功返回的结果，包含ETag等信息. """ headers = mapped(kwargs) - if 'Metadata' in headers.keys(): - for i in headers['Metadata'].keys(): - headers[i] = headers['Metadata'][i] - headers.pop('Metadata') - - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?append&position="+str(Position)) + params = {'append': '', 'position': Position} + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("append object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -2145,9 +2189,10 @@ def append_object(self, Bucket, Key, Position, Data, **kwargs): rt = self.send_request( method='POST', url=url, - auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key), + auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key, Key, params=params), data=Body, - headers=headers) + headers=headers, + params=params) response = rt.headers return response @@ -2171,7 +2216,7 @@ def put_object_from_local_file(self, Bucket, LocalFilePath, Key, EnableMD5=False LocalFilePath='local.txt', Key='test.txt' ) - print response['ETag'] + print (response['ETag']) """ with open(LocalFilePath, 'rb') as fp: return self.put_object(Bucket, fp, Key, EnableMD5, **kwargs) diff --git a/qcloud_cos/cos_comm.py b/qcloud_cos/cos_comm.py index ae7c6f72..748e3955 100644 --- a/qcloud_cos/cos_comm.py +++ b/qcloud_cos/cos_comm.py @@ -1,5 +1,7 @@ # -*- coding=utf-8 +from six import text_type, binary_type, string_types +from six.moves.urllib.parse import quote, unquote import hashlib import base64 import os @@ -8,12 +10,10 @@ import xml.dom.minidom import xml.etree.ElementTree from datetime import datetime -from urllib import quote -from urllib import unquote -from xml2dict import Xml2Dict from dicttoxml import dicttoxml -from cos_exception import CosClientError -from cos_exception import CosServiceError +from .xml2dict import Xml2Dict +from .cos_exception import CosClientError +from .cos_exception import CosServiceError SINGLE_UPLOAD_LENGTH = 5*1024*1024*1024 # 单次上传文件最大为5G LOGGING_UIN = 'id="qcs::cam::uin/100001001014:uin/100001001014"' @@ -58,36 +58,53 @@ def to_unicode(s): - if isinstance(s, unicode): - return s - else: - return s.decode('utf-8') + """将字符串转为unicode""" + if isinstance(s, binary_type): + try: + return s.decode('utf-8') + except UnicodeDecodeError as e: + raise CosClientError('your bytes strings can not be decoded in utf8, utf8 support only!') + return s + + +def to_bytes(s): + """将字符串转为bytes""" + if isinstance(s, text_type): + try: + return s.encode('utf-8') + except UnicodeEncodeError as e: + raise CosClientError('your unicode strings can not encoded in utf8, utf8 support only!') + return s +def format_bytes(): + """将需要传输的内容转换为bytes, body以及header的values""" def get_raw_md5(data): + """计算md5 md5的输入必须为bytes""" + data = to_bytes(data) m2 = hashlib.md5(data) etag = '"' + str(m2.hexdigest()) + '"' return etag def get_md5(data): + """计算 base64 md5 md5的输入必须为bytes""" + data = to_bytes(data) m2 = hashlib.md5(data) MD5 = base64.standard_b64encode(m2.digest()) return MD5 def get_content_md5(body): + """计算任何输入流的md5值""" body_type = type(body) - if body_type == str: + if body_type == string_types: return get_md5(body) - elif body_type == file: - if hasattr(body, 'tell') and hasattr(body, 'seek') and hasattr(body, 'read'): - file_position = body.tell() # 记录文件当前位置 - md5_str = get_md5(body.read()) - body.seek(file_position) # 恢复初始的文件位置 - return md5_str - else: - raise CosClientError('can not get md5 digest for file without necessary attrs, including tell, seek and read') + elif hasattr(body, 'tell') and hasattr(body, 'seek') and hasattr(body, 'read'): + file_position = body.tell() # 记录文件当前位置 + md5_str = get_md5(body.read()) + body.seek(file_position) # 恢复初始的文件位置 + return md5_str return None @@ -97,19 +114,19 @@ def dict_to_xml(data): root = doc.createElement('CompleteMultipartUpload') doc.appendChild(root) - if 'Part' not in data.keys(): + if 'Part' not in data: raise CosClientError("Invalid Parameter, Part Is Required!") for i in data['Part']: nodePart = doc.createElement('Part') - if 'PartNumber' not in i.keys(): + if 'PartNumber' not in i: raise CosClientError("Invalid Parameter, PartNumber Is Required!") nodeNumber = doc.createElement('PartNumber') nodeNumber.appendChild(doc.createTextNode(str(i['PartNumber']))) - if 'ETag' not in i.keys(): + if 'ETag' not in i: raise CosClientError("Invalid Parameter, ETag Is Required!") nodeETag = doc.createElement('ETag') @@ -148,101 +165,122 @@ def get_id_from_xml(data, name): def mapped(headers): """S3到COS参数的一个映射""" _headers = dict() - for i in headers.keys(): + for i in headers: if i in maplist: - _headers[maplist[i]] = headers[i] + if i == 'Metadata': + for meta in headers[i]: + _headers[meta] = headers[i][meta] + else: + _headers[maplist[i]] = headers[i] else: - raise CosClientError('No Parameter Named '+i+' Please Check It') + raise CosClientError('No Parameter Named ' + i + ' Please Check It') return _headers def format_xml(data, root, lst=list()): - """将dict转换为xml""" + """将dict转换为xml, xml_config是一个bytes""" xml_config = dicttoxml(data, item_func=lambda x: x, custom_root=root, attr_type=False) for i in lst: - xml_config = xml_config.replace(i+i, i) + xml_config = xml_config.replace(to_bytes(i+i), to_bytes(i)) return xml_config +def format_values(data): + """格式化headers和params中的values为bytes""" + for i in data: + data[i] = to_bytes(data[i]) + return data + + def format_region(region): """格式化地域""" + if not isinstance(region, string_types): + raise CosClientError("bucket is not string type") if not region: raise CosClientError("region is required not empty!") - if region.find('cos.') != -1: + region = to_unicode(region) + if region.find(u'cos.') != -1: return region # 传入cos.ap-beijing-1这样显示加上cos.的region - if region == 'cn-north' or region == 'cn-south' or region == 'cn-east' or region == 'cn-south-2' or region == 'cn-southwest' or region == 'sg': + if region == u'cn-north' or region == u'cn-south' or region == u'cn-east' or region == u'cn-south-2' or region == u'cn-southwest' or region == u'sg': return region # 老域名不能加cos. # 支持v4域名映射到v5 - if region == 'cossh': - return 'cos.ap-shanghai' - if region == 'cosgz': - return 'cos.ap-guangzhou' + if region == u'cossh': + return u'cos.ap-shanghai' + if region == u'cosgz': + return u'cos.ap-guangzhou' if region == 'cosbj': - return 'cos.ap-beijing' + return u'cos.ap-beijing' if region == 'costj': - return 'cos.ap-beijing-1' - if region == 'coscd': - return 'cos.ap-chengdu' - if region == 'cossgp': - return 'cos.ap-singapore' - if region == 'coshk': - return 'cos.ap-hongkong' - if region == 'cosca': - return 'cos.na-toronto' - if region == 'cosger': - return 'cos.eu-frankfurt' + return u'cos.ap-beijing-1' + if region == u'coscd': + return u'cos.ap-chengdu' + if region == u'cossgp': + return u'cos.ap-singapore' + if region == u'coshk': + return u'cos.ap-hongkong' + if region == u'cosca': + return u'cos.na-toronto' + if region == u'cosger': + return u'cos.eu-frankfurt' - return 'cos.' + region # 新域名加上cos. + return u'cos.' + region # 新域名加上cos. def format_bucket(bucket, appid): """兼容新老bucket长短命名,appid为空默认为长命名,appid不为空则认为是短命名""" - if not isinstance(bucket, str): - raise CosClientError("bucket is not str") + if not isinstance(bucket, string_types): + raise CosClientError("bucket is not string") + if not bucket: + raise CosClientError("bucket is required not empty") # appid为空直接返回bucket if not appid: - return bucket + return to_unicode(bucket) + if not isinstance(appid, string_types): + raise CosClientError("appid is not string") + bucket = to_unicode(bucket) + appid = to_unicode(appid) # appid不为空,检查是否以-appid结尾 - if bucket.endswith("-"+appid): + if bucket.endswith(u"-"+appid): return bucket - return bucket + "-" + appid + return bucket + u"-" + appid def format_path(path): """检查path是否合法,格式化path""" - if not isinstance(path, str): - raise CosClientError("your Key is not str") - if path == "": - raise CosClientError("Key can't be empty string") - if path[0] == '/': + if not isinstance(path, string_types): + raise CosClientError("key is not string") + if not path: + raise CosClientError("Key is required not empty") + path = to_unicode(path) + if path[0] == u'/': path = path[1:] # 提前对path进行encode - path = quote(path, '/-_.~') + path = quote(to_bytes(path), b'/-_.~') return path def get_copy_source_info(CopySource): """获取拷贝源的所有信息""" - appid = "" - versionid = "" - if 'Appid' in CopySource.keys(): + appid = u"" + versionid = u"" + if 'Appid' in CopySource: appid = CopySource['Appid'] - if 'Bucket' in CopySource.keys(): + if 'Bucket' in CopySource: bucket = CopySource['Bucket'] bucket = format_bucket(bucket, appid) else: raise CosClientError('CopySource Need Parameter Bucket') - if 'Region' in CopySource.keys(): + if 'Region' in CopySource: region = CopySource['Region'] region = format_region(region) else: raise CosClientError('CopySource Need Parameter Region') - if 'Key' in CopySource.keys(): - path = CopySource['Key'] + if 'Key' in CopySource: + path = to_unicode(CopySource['Key']) else: raise CosClientError('CopySource Need Parameter Key') - if 'VersionId' in CopySource.keys(): - versionid = CopySource['VersionId'] + if 'VersionId' in CopySource: + versionid = to_unicode(CopySource['VersionId']) return bucket, path, region, versionid @@ -250,9 +288,9 @@ def gen_copy_source_url(CopySource): """拼接拷贝源url""" bucket, path, region, versionid = get_copy_source_info(CopySource) path = format_path(path) - if versionid != '': - path = path + '?versionId=' + versionid - url = "{bucket}.{region}.myqcloud.com/{path}".format( + if versionid != u'': + path = path + u'?versionId=' + versionid + url = u"{bucket}.{region}.myqcloud.com/{path}".format( bucket=bucket, region=region, path=path @@ -262,9 +300,9 @@ def gen_copy_source_url(CopySource): def gen_copy_source_range(begin_range, end_range): """拼接bytes=begin-end形式的字符串""" - range = "bytes={first}-{end}".format( - first=begin_range, - end=end_range + range = u"bytes={first}-{end}".format( + first=to_unicode(begin_range), + end=to_unicode(end_range) ) return range @@ -272,9 +310,9 @@ def gen_copy_source_range(begin_range, end_range): def check_object_content_length(data): """put_object接口和upload_part接口的文件大小不允许超过5G""" content_len = 0 - if type(data) is str: - content_len = len(data) - elif type(data) is file and hasattr(data, 'fileno') and hasattr(data, 'tell'): + if type(data) is string_types: + content_len = len(to_bytes(data)) + elif hasattr(data, 'fileno') and hasattr(data, 'tell'): fileno = data.fileno() total_length = os.fstat(fileno).st_size current_position = data.tell() @@ -292,9 +330,9 @@ def deal_with_empty_file_stream(data): total_length = os.fstat(fileno).st_size current_position = data.tell() if total_length - current_position == 0: - return "" + return b"" except io.UnsupportedOperation: - return "" + return b"" return data @@ -302,7 +340,7 @@ def format_dict(data, key_lst): """转换返回dict中的可重复字段为list""" for key in key_lst: # 将dict转为list，保持一致 - if key in data.keys() and isinstance(data[key], dict): + if key in data and isinstance(data[key], dict): lst = [] lst.append(data[key]) data[key] = lst @@ -312,12 +350,12 @@ def format_dict(data, key_lst): def decode_result(data, key_lst, multi_key_list): """decode结果中的字段""" for key in key_lst: - if key in data.keys() and data[key]: + if key in data and data[key]: data[key] = unquote(data[key]) for multi_key in multi_key_list: - if multi_key[0] in data.keys(): + if multi_key[0] in data: for item in data[multi_key[0]]: - if multi_key[1] in item.keys() and item[multi_key[1]]: + if multi_key[1] in item and item[multi_key[1]]: item[multi_key[1]] = unquote(item[multi_key[1]]) return data diff --git a/qcloud_cos/cos_threadpool.py b/qcloud_cos/cos_threadpool.py index d302eb41..2127c4a9 100644 --- a/qcloud_cos/cos_threadpool.py +++ b/qcloud_cos/cos_threadpool.py @@ -2,7 +2,7 @@ from threading import Thread from logging import getLogger -from Queue import Queue +from six.moves import queue from threading import Lock import gc logger = getLogger(__name__) @@ -94,12 +94,12 @@ def raise_exception(): raise ValueError("Pa! Exception!") for i in range(1000): pool.add_task(task_sleep, 0.001) - print i + print (i) pool.add_task(task_sleep, 0) pool.add_task(task_sleep, 0) # pool.add_task(raise_exception) # pool.add_task(raise_exception) pool.wait_completion() - print pool.get_result() + print (pool.get_result()) # [(1, 0, ['hello, sleep 5 seconds']), (2, 1, ['hello, sleep 2 seconds', 'hello, sleep 3 seconds', ValueError('Pa! Exception!',)])] diff --git a/qcloud_cos/demo.py b/qcloud_cos/demo.py index 3d016146..b3d9c910 100644 --- a/qcloud_cos/demo.py +++ b/qcloud_cos/demo.py @@ -35,7 +35,7 @@ CacheControl='no-cache', ContentDisposition='download.txt' ) - print response['ETag'] + print (response['ETag']) # 字节流 简单上传 response = client.put_object( @@ -45,7 +45,7 @@ CacheControl='no-cache', ContentDisposition='download.txt' ) -print response['ETag'] +print (response['ETag']) # 文件下载 获取文件到本地 response = client.get_object( @@ -60,7 +60,7 @@ Key=file_name, ) fp = response['Body'].get_raw_stream() -print fp.read(2) +print (fp.read(2)) # 文件下载 捕获异常 try: @@ -69,13 +69,13 @@ Key='not_exist.txt', ) fp = response['Body'].get_raw_stream() - print fp.read(2) + print (fp.read(2)) except CosServiceError as e: - print e.get_origin_msg() - print e.get_digest_msg() - print e.get_status_code() - print e.get_error_code() - print e.get_error_msg() - print e.get_resource_location() - print e.get_trace_id() - print e.get_request_id() + print (e.get_origin_msg()) + print (e.get_digest_msg()) + print (e.get_status_code()) + print (e.get_error_code()) + print (e.get_error_msg()) + print (e.get_resource_location()) + print (e.get_trace_id()) + print (e.get_request_id()) diff --git a/qcloud_cos/xml2dict.py b/qcloud_cos/xml2dict.py index dedd5c14..223a3b94 100644 --- a/qcloud_cos/xml2dict.py +++ b/qcloud_cos/xml2dict.py @@ -19,7 +19,7 @@ def __init__(self, parent_node): self.updateDict({element.tag: element.text}) def updateDict(self, aDict): - for key in aDict.keys(): + for key in aDict: if key in self: value = self.pop(key) if type(value) is not list: @@ -43,4 +43,4 @@ def updateDict(self, aDict): """ root = xml.etree.ElementTree.fromstring(s) xmldict = Xml2Dict(root) - print xmldict + print (xmldict) diff --git a/requirements.txt b/requirements.txt index 1cbfa8ca..98dbdc12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests dicttoxml +six diff --git a/ut/test.py b/ut/test.py index b835c13d..12d379d8 100644 --- a/ut/test.py +++ b/ut/test.py @@ -37,22 +37,23 @@ def gen_file(path, size): def print_error_msg(e): - print e.get_origin_msg() - print e.get_digest_msg() - print e.get_status_code() - print e.get_error_code() - print e.get_error_msg() - print e.get_resource_location() - print e.get_trace_id() - print e.get_request_id() + print (e.get_origin_msg()) + print (e.get_digest_msg()) + print (e.get_status_code()) + print (e.get_error_code()) + print (e.get_error_msg()) + print (e.get_resource_location()) + print (e.get_trace_id()) + print (e.get_request_id()) def setUp(): - print "start test..." + print ("start test...") + print (sys.version_info) def tearDown(): - print "function teardown" + print ("function teardown") def test_put_get_delete_object_10MB(): @@ -376,7 +377,7 @@ def test_get_presigned_url(): Key='中文.txt' ) assert url - print url + print (url) def test_get_bucket_location(): @@ -496,7 +497,7 @@ def test_put_get_delete_replication(): { 'ID': '123', 'Status': 'Enabled', - 'Prefix': 'replication', + 'Prefix': '/中文', 'Destination': { 'Bucket': 'qcs:id/0:cos:cn-south:appid/1252448703:replicationsouth' } @@ -577,7 +578,7 @@ def test_upload_file_multithreading(): ed = time.time() # 记录结束时间 if os.path.exists(file_name): os.remove(file_name) - print ed - st + print (ed - st) def test_copy_file_automatically(): @@ -663,7 +664,7 @@ def test_put_get_bucket_logging(): response = logging_client.get_bucket_logging( Bucket=logging_bucket ) - print response + print (response) assert response['LoggingEnabled']['TargetBucket'] == logging_bucket assert response['LoggingEnabled']['TargetPrefix'] == 'test' From 7d1a4cdbe06c0bc7361d30a434df78f8d28ee503 Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 28 Apr 2018 21:53:20 +0800 Subject: [PATCH 2/9] fix pep8 --- qcloud_cos/cos_client.py | 17 +++++++---------- qcloud_cos/cos_comm.py | 2 -- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 413ff74d..08c4a3d9 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) + class CosConfig(object): """config类，保存用户相关信息""" def __init__(self, Appid=None, Region=None, Secret_id=None, Secret_key=None, Token=None, Scheme=None, Timeout=None, Access_id=None, Access_key=None): @@ -61,7 +62,6 @@ def __init__(self, Appid=None, Region=None, Secret_id=None, Secret_key=None, Tok else: raise CosClientError('SecretId and SecretKey is Required!') - def uri(self, bucket, path=None, scheme=None, region=None): """拼接url @@ -153,8 +153,8 @@ def send_request(self, method, url, timeout=30, **kwargs): kwargs['headers'] = format_values(kwargs['headers']) if 'data' in kwargs: kwargs['data'] = to_bytes(kwargs['data']) - #try: - for j in range(self._retry): + try: + for j in range(self._retry): if method == 'POST': res = self._session.post(url, timeout=timeout, **kwargs) elif method == 'GET': @@ -167,9 +167,9 @@ def send_request(self, method, url, timeout=30, **kwargs): res = self._session.head(url, timeout=timeout, **kwargs) if res.status_code < 400: # 2xx和3xx都认为是成功的 return res - #except Exception as e: # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误 - #logger.exception('url:%s, exception:%s' % (url, str(e))) - #raise CosClientError(str(e)) + except Exception as e: # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误 + logger.exception('url:%s, exception:%s' % (url, str(e))) + raise CosClientError(str(e)) if res.status_code >= 400: # 所有的4XX,5XX都认为是COSServiceError if method == 'HEAD' and res.status_code == 404: # Head 需要处理 @@ -262,13 +262,11 @@ def get_object(self, Bucket, Key, **kwargs): params[key] = headers[key] else: final_headers[key] = headers[key] - headers = final_headers if 'versionId' in headers: params['versionId'] = headers['versionId'] del headers['versionId'] - params = format_values(params) url = self._conf.uri(bucket=Bucket, path=Key) @@ -717,7 +715,6 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, Part decodeflag = False else: params['encoding-type'] = 'url' - params = format_values(params) url = self._conf.uri(bucket=Bucket, path=Key) logger.info("list multipart upload parts, url=:{url} ,headers=:{headers}".format( @@ -2180,7 +2177,7 @@ def append_object(self, Bucket, Key, Position, Data, **kwargs): :return(dict): 上传成功返回的结果，包含ETag等信息. """ headers = mapped(kwargs) - params = {'append': '', 'position': Position} + params = {'append': '', 'position': Position} url = self._conf.uri(bucket=Bucket, path=Key) logger.info("append object, url=:{url} ,headers=:{headers}".format( url=url, diff --git a/qcloud_cos/cos_comm.py b/qcloud_cos/cos_comm.py index 748e3955..70f8c563 100644 --- a/qcloud_cos/cos_comm.py +++ b/qcloud_cos/cos_comm.py @@ -77,8 +77,6 @@ def to_bytes(s): return s -def format_bytes(): - """将需要传输的内容转换为bytes, body以及header的values""" def get_raw_md5(data): """计算md5 md5的输入必须为bytes""" data = to_bytes(data) From b8dd682c3598f7182ff257c11a41895b8a94344a Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 28 Apr 2018 21:53:57 +0800 Subject: [PATCH 3/9] add py3 travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 145a1396..bd9103b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,11 @@ language: python python: - '2.6' - '2.7' +- '3.5' +- '3.6' install: - pip install requests +- pip install six - pip install nose - pip install pep8 - pip install dicttoxml From 39ac904d6ae8f8864ebd02a4c692c695462e60ea Mon Sep 17 00:00:00 2001 From: tiedu Date: Sat, 28 Apr 2018 22:02:27 +0800 Subject: [PATCH 4/9] fix py3 pep8 --- qcloud_cos/cos_threadpool.py | 4 ++-- qcloud_cos/demo.py | 24 ++++++++++++------------ qcloud_cos/xml2dict.py | 3 ++- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/qcloud_cos/cos_threadpool.py b/qcloud_cos/cos_threadpool.py index 2127c4a9..2ec1903d 100644 --- a/qcloud_cos/cos_threadpool.py +++ b/qcloud_cos/cos_threadpool.py @@ -94,12 +94,12 @@ def raise_exception(): raise ValueError("Pa! Exception!") for i in range(1000): pool.add_task(task_sleep, 0.001) - print (i) + print(i) pool.add_task(task_sleep, 0) pool.add_task(task_sleep, 0) # pool.add_task(raise_exception) # pool.add_task(raise_exception) pool.wait_completion() - print (pool.get_result()) + print(pool.get_result()) # [(1, 0, ['hello, sleep 5 seconds']), (2, 1, ['hello, sleep 2 seconds', 'hello, sleep 3 seconds', ValueError('Pa! Exception!',)])] diff --git a/qcloud_cos/demo.py b/qcloud_cos/demo.py index b3d9c910..c886c179 100644 --- a/qcloud_cos/demo.py +++ b/qcloud_cos/demo.py @@ -35,7 +35,7 @@ CacheControl='no-cache', ContentDisposition='download.txt' ) - print (response['ETag']) + print(response['ETag']) # 字节流 简单上传 response = client.put_object( @@ -45,7 +45,7 @@ CacheControl='no-cache', ContentDisposition='download.txt' ) -print (response['ETag']) +print(response['ETag']) # 文件下载 获取文件到本地 response = client.get_object( @@ -60,7 +60,7 @@ Key=file_name, ) fp = response['Body'].get_raw_stream() -print (fp.read(2)) +print(fp.read(2)) # 文件下载 捕获异常 try: @@ -69,13 +69,13 @@ Key='not_exist.txt', ) fp = response['Body'].get_raw_stream() - print (fp.read(2)) + print(fp.read(2)) except CosServiceError as e: - print (e.get_origin_msg()) - print (e.get_digest_msg()) - print (e.get_status_code()) - print (e.get_error_code()) - print (e.get_error_msg()) - print (e.get_resource_location()) - print (e.get_trace_id()) - print (e.get_request_id()) + print(e.get_origin_msg()) + print(e.get_digest_msg()) + print(e.get_status_code()) + print(e.get_error_code()) + print(e.get_error_msg()) + print(e.get_resource_location()) + print(e.get_trace_id()) + print(e.get_request_id()) diff --git a/qcloud_cos/xml2dict.py b/qcloud_cos/xml2dict.py index 223a3b94..334d3c96 100644 --- a/qcloud_cos/xml2dict.py +++ b/qcloud_cos/xml2dict.py @@ -3,6 +3,7 @@ class Xml2Dict(dict): + def __init__(self, parent_node): if parent_node.items(): self.updateDict(dict(parent_node.items())) @@ -43,4 +44,4 @@ def updateDict(self, aDict): """ root = xml.etree.ElementTree.fromstring(s) xmldict = Xml2Dict(root) - print (xmldict) + print(xmldict) From f78355a98bdd4802f55c63d00b2f5514251f34b1 Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 2 May 2018 18:07:43 +0800 Subject: [PATCH 5/9] Modify upload_file --- qcloud_cos/cos_client.py | 23 +++++++++++------ qcloud_cos/cos_threadpool.py | 2 +- qcloud_cos/demo.py | 2 +- ut/test.py | 49 +++++++++++++++++++++++++++++------- 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 08c4a3d9..a39b89d3 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -7,6 +7,7 @@ import os import sys import copy +import time import xml.dom.minidom import xml.etree.ElementTree from requests import Request, Session @@ -270,9 +271,10 @@ def get_object(self, Bucket, Key, **kwargs): params = format_values(params) url = self._conf.uri(bucket=Bucket, path=Key) - logger.info("get object, url=:{url} ,headers=:{headers}".format( + logger.info("get object, url=:{url} ,headers=:{headers}, params=:{params}".format( url=url, - headers=headers)) + headers=headers, + params=params)) rt = self.send_request( method='GET', url=url, @@ -583,9 +585,10 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, EnableMD5=False, params = {'partNumber': PartNumber, 'uploadId': UploadId} params = format_values(params) url = self._conf.uri(bucket=Bucket, path=Key) - logger.info("upload part, url=:{url} ,headers=:{headers}".format( + logger.info("upload part, url=:{url} ,headers=:{headers}, params=:{params}".format( url=url, - headers=headers)) + headers=headers, + params=params)) Body = deal_with_empty_file_stream(Body) if EnableMD5: md5_str = get_content_md5(Body) @@ -1874,11 +1877,13 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num, UploadId=uploadid, PartNumberMarker=part_number_marker ) - parts_info.extend(response['Part']) + # 已经存在的分块上传,有可能一个分块都没有上传,判断一下 + if 'Part' in response: + parts_info.extend(response['Part']) if response['IsTruncated'] == 'false': list_over_status = True else: - part_number_marker = int(response['NextMarker']) + part_number_marker = int(response['NextPartNumberMarker']) for part in parts_info: part_num = int(part['PartNumber']) # 如果分块数量大于本地计算出的最大数量,校验失败 @@ -1934,6 +1939,8 @@ def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, **kwa if last_size != 0: parts_num += 1 + else: # 如果刚好整除,最后一块的大小等于分块大小 + last_size = part_size if parts_num > 10000: parts_num = 10000 part_size = file_size // parts_num @@ -1946,12 +1953,14 @@ def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, **kwa already_exist_parts = {} uploadid = self._get_resumable_uploadid(Bucket, Key) if uploadid is not None: + logger.info("fetch an existed uploadid in remote cos, uploadid={uploadid}".format(uploadid=uploadid)) # 校验服务端返回的每个块的信息是否和本地的每个块的信息相同,只有校验通过的情况下才可以进行断点续传 resumable_flag = self._check_all_upload_parts(Bucket, Key, uploadid, LocalFilePath, parts_num, part_size, last_size, already_exist_parts) # 如果不能断点续传,则创建一个新的分块上传 if not resumable_flag: rt = self.create_multipart_upload(Bucket=Bucket, Key=Key, **kwargs) uploadid = rt['UploadId'] + logger.info("create a new uploadid in upload_file, uploadid={uploadid}".format(uploadid=uploadid)) # 上传分块 offset = 0 # 记录文件偏移量 @@ -1968,7 +1977,7 @@ def upload_file(self, Bucket, Key, LocalFilePath, PartSize=1, MAXThread=5, **kwa pool.wait_completion() result = pool.get_result() if not result['success_all'] or len(lst) != parts_num: - raise CosClientError('some upload_part fail after max_retry') + raise CosClientError('some upload_part fail after max_retry, please upload_file again') lst = sorted(lst, key=lambda x: x['PartNumber']) # 按PartNumber升序排列 # 完成分块上传 diff --git a/qcloud_cos/cos_threadpool.py b/qcloud_cos/cos_threadpool.py index 2ec1903d..1900d69f 100644 --- a/qcloud_cos/cos_threadpool.py +++ b/qcloud_cos/cos_threadpool.py @@ -2,7 +2,7 @@ from threading import Thread from logging import getLogger -from six.moves import queue +from six.moves.queue import Queue from threading import Lock import gc logger = getLogger(__name__) diff --git a/qcloud_cos/demo.py b/qcloud_cos/demo.py index c886c179..cac39b20 100644 --- a/qcloud_cos/demo.py +++ b/qcloud_cos/demo.py @@ -13,7 +13,7 @@ # cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224 -logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) +logging.basicConfig(level=logging.INFO, stream=sys.stdout) # 设置用户属性, 包括secret_id, secret_key, region # appid已在配置中移除,请在参数Bucket中带上appid。Bucket由bucketname-appid组成 diff --git a/ut/test.py b/ut/test.py index 12d379d8..937496b2 100644 --- a/ut/test.py +++ b/ut/test.py @@ -12,7 +12,8 @@ SECRET_ID = os.environ["SECRET_ID"] SECRET_KEY = os.environ["SECRET_KEY"] -test_bucket = "test01-1252448703" +TRAVIS_FLAG = os.environ["TRAVIS_FLAG"] +test_bucket = 'cos-python-v5-test-' + str(sys.version_info[0]) + '-' + str(sys.version_info[1]) + '-' + '1252448703' test_object = "test.txt" special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" conf = CosConfig( @@ -23,6 +24,19 @@ client = CosS3Client(conf) +def _create_test_bucket(test_bucket): + try: + response = client.create_bucket( + Bucket=test_bucket, + ) + except Exception as e: + if e.get_error_code() == 'BucketAlreadyOwnedByYou': + print('BucketAlreadyOwnedByYou') + else: + raise e + return None + + def get_raw_md5(data): m2 = hashlib.md5(data) etag = '"' + str(m2.hexdigest()) + '"' @@ -49,7 +63,8 @@ def print_error_msg(e): def setUp(): print ("start test...") - print (sys.version_info) + print ("start create bucket " + test_bucket) + _create_test_bucket(test_bucket) def tearDown(): @@ -147,6 +162,11 @@ def test_put_object_non_exist_bucket(): def test_put_object_acl(): """设置object acl""" + response = client.put_object( + Bucket=test_bucket, + Key=test_object, + Body='test acl' + ) response = client.put_object_acl( Bucket=test_bucket, Key=test_object, @@ -161,6 +181,10 @@ def test_get_object_acl(): Key=test_object ) assert response + response = client.delete_object( + Bucket=test_bucket, + Key=test_object + ) def test_copy_object_diff_bucket(): @@ -353,7 +377,7 @@ def test_get_bucket_acl_normal(): def test_list_objects(): """列出bucket下的objects""" response = client.list_objects( - Bucket=test_bucket, + Bucket='test01-1252448703', MaxKeys=100, Prefix='中文', Delimiter='/' @@ -497,7 +521,7 @@ def test_put_get_delete_replication(): { 'ID': '123', 'Status': 'Enabled', - 'Prefix': '/中文', + 'Prefix': '中文', 'Destination': { 'Bucket': 'qcs:id/0:cos:cn-south:appid/1252448703:replicationsouth' } @@ -517,7 +541,7 @@ def test_put_get_delete_replication(): Bucket=test_bucket ) assert response - # delete lifecycle + # delete replication response = client.delete_bucket_replication( Bucket=test_bucket ) @@ -565,7 +589,10 @@ def test_list_multipart_uploads(): def test_upload_file_multithreading(): """根据文件大小自动选择分块大小,多线程并发上传提高上传速度""" file_name = "thread_1GB" - gen_file(file_name, 5) # set 5MB beacuse travis too slow + file_size = 1024 + if TRAVIS_FLAG == 'true': + file_size = 5 # set 5MB beacuse travis too slow + gen_file(file_name, file_size) st = time.time() # 记录开始时间 response = client.upload_file( Bucket=test_bucket, @@ -626,7 +653,8 @@ def test_use_get_auth(): Key='test.txt', Params={'acl': '', 'unsed': '123'} ) - response = requests.get('http://test01-1252448703.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123', headers={'Authorization': auth}) + url = 'http://' + test_bucket + '.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123' + response = requests.get(url, headers={'Authorization': auth}) assert response.status_code == 200 @@ -671,9 +699,8 @@ def test_put_get_bucket_logging(): def test_put_object_enable_md5(): """上传文件,SDK计算content-md5头部""" - file_size = 10 file_name = 'test_object_sdk_caculate_md5.file' - gen_file(file_name, 10) + gen_file(file_name, 1) with open(file_name, 'rb') as f: etag = get_raw_md5(f.read()) with open(file_name, 'rb') as fp: @@ -704,6 +731,10 @@ def test_put_object_from_local_file(): Key=file_name ) assert put_response['ETag'] == etag + response = client.delete_object( + Bucket=test_bucket, + Key=file_name + ) if os.path.exists(file_name): os.remove(file_name) From 86445d2105d87f978158d3d3d058a49513357dc5 Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 2 May 2018 21:16:38 +0800 Subject: [PATCH 6/9] modify upload_file to fetch the last uploadid and demo --- qcloud_cos/cos_auth.py | 2 +- qcloud_cos/cos_client.py | 9 ++++-- qcloud_cos/demo.py | 67 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 887ed614..3ebc6fb7 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -49,7 +49,7 @@ def __call__(self, r): format_str = u"{method}\n{host}\n{params}\n{headers}\n".format( method=r.method.lower(), host=path, - params=urlencode(sorted(uri_params.items())), + params=urlencode(sorted(uri_params.items())).replace('+', '%20'), headers='&'.join(map(lambda tupl: "%s=%s" % (tupl[0], tupl[1]), sorted(headers.items()))) ) logger.debug("format str: " + format_str) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index a39b89d3..e1b8201b 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -1830,9 +1830,12 @@ def _get_resumable_uploadid(self, bucket, key): Prefix=key ) if 'Upload' in multipart_response: - if multipart_response['Upload'][0]['Key'] == key: - return multipart_response['Upload'][0]['UploadId'] - + # 取最后一个(最新的)uploadid + index = len(multipart_response['Upload']) - 1 + while index >= 0: + if multipart_response['Upload'][index]['Key'] == key: + return multipart_response['Upload'][index]['UploadId'] + index -= 1 return None def _check_single_upload_part(self, local_path, offset, local_part_size, remote_part_size, remote_etag): diff --git a/qcloud_cos/demo.py b/qcloud_cos/demo.py index cac39b20..5b18f5d1 100644 --- a/qcloud_cos/demo.py +++ b/qcloud_cos/demo.py @@ -7,7 +7,7 @@ import sys import logging -# 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7 +# 腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7以及Python3.x # pip安装指南:pip install -U cos-python-sdk-v5 @@ -32,21 +32,57 @@ Body=fp, Key=file_name, StorageClass='STANDARD', - CacheControl='no-cache', - ContentDisposition='download.txt' + ContentType='text/html; charset=utf-8' ) print(response['ETag']) # 字节流 简单上传 response = client.put_object( Bucket='test04-123456789', - Body='abcdefg', + Body=b'abcdefg', + Key=file_name +) +print(response['ETag']) + +# 本地路径 简单上传 +response = client.put_object_from_local_file( + Bucket='test04-123456789', + LocalFilePath='local.txt', Key=file_name, - CacheControl='no-cache', - ContentDisposition='download.txt' ) print(response['ETag']) +# 设置HTTP头部 简单上传 +response = client.put_object( + Bucket='test04-123456789', + Body=b'test', + Key=file_name, + ContentType='text/html; charset=utf-8' +) +print(response['ETag']) + +# 设置自定义头部 简单上传 +response = client.put_object( + Bucket='test04-123456789', + Body=b'test', + Key=file_name, + Metadata={ + 'x-cos-meta-key1': 'value1', + 'x-cos-meta-key2': 'value2' + } +) +print(response['ETag']) + +# 高级上传接口(推荐) +response = client.upload_file( + Bucket='test04-123456789', + LocalFilePath='local.txt', + Key=file_name, + PartSize=10, + MAXThread=10 +) +print response['ETag'] + # 文件下载 获取文件到本地 response = client.get_object( Bucket='test04-123456789', @@ -62,6 +98,25 @@ fp = response['Body'].get_raw_stream() print(fp.read(2)) +# 文件下载 设置Response HTTP 头部 +response = client.get_object( + Bucket='test04-123456789', + Key=file_name, + ResponseContentType='text/html; charset=utf-8' +) +print response['Content-Type'] +fp = response['Body'].get_raw_stream() +print(fp.read(2)) + +# 文件下载 指定下载范围 +response = client.get_object( + Bucket='test04-123456789', + Key=file_name, + Range='bytes=0-10' +) +fp = response['Body'].get_raw_stream() +print(fp.read()) + # 文件下载 捕获异常 try: response = client.get_object( From efb8efd214d1f9c4fff99cc0437ff68581157d25 Mon Sep 17 00:00:00 2001 From: tiedu Date: Wed, 2 May 2018 21:31:22 +0800 Subject: [PATCH 7/9] fix code style --- qcloud_cos/cos_client.py | 1 - qcloud_cos/xml2dict.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index e1b8201b..900680f1 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -7,7 +7,6 @@ import os import sys import copy -import time import xml.dom.minidom import xml.etree.ElementTree from requests import Request, Session diff --git a/qcloud_cos/xml2dict.py b/qcloud_cos/xml2dict.py index 334d3c96..3496fedf 100644 --- a/qcloud_cos/xml2dict.py +++ b/qcloud_cos/xml2dict.py @@ -34,6 +34,7 @@ def updateDict(self, aDict): else: self.update({key: aDict[key]}) + if __name__ == "__main__": s = """ From 42df6ce09a0e6235b6b2bc27e4e99210f4340d95 Mon Sep 17 00:00:00 2001 From: tiedu Date: Tue, 8 May 2018 15:28:47 +0800 Subject: [PATCH 8/9] fix restore --- qcloud_cos/cos_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 900680f1..2fc714c9 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -827,7 +827,7 @@ def restore_object(self, Bucket, Key, RestoreRequest={}, **kwargs): if 'versionId' in headers: params['versionId'] = headers['versionId'] headers.pop('versionId') - url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?restore") + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("restore_object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) From c5e5ed75fff128328897219e4ab601792af9ebaf Mon Sep 17 00:00:00 2001 From: tiedu Date: Tue, 8 May 2018 17:21:12 +0800 Subject: [PATCH 9/9] modify UA to 5.1.5.0 --- qcloud_cos/cos_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 2fc714c9..e3cbcf29 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -149,7 +149,7 @@ def send_request(self, method, url, timeout=30, **kwargs): timeout = self._conf._timeout if self._conf._token is not None: kwargs['headers']['x-cos-security-token'] = self._conf._token - kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.1' + kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.5.0' kwargs['headers'] = format_values(kwargs['headers']) if 'data' in kwargs: kwargs['data'] = to_bytes(kwargs['data'])