diff --git a/README.md b/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..a2a143ea --- /dev/null +++ b/README.rst @@ -0,0 +1,131 @@ +Qcloud COSv5 SDK +####################### + +介绍 +_______ + +腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7。 + +安装指南 +__________ + +使用pip安装 :: + + pip install -U qcloud_cos_v5 + + +手动安装:: + + python setup.py install + +使用方法 +__________ + +使用python sdk，参照https://github.com/tencentyun/cos-python-sdk-v5/blob/master/qcloud_cos/test.py + +.. code:: python + + # 设置用户属性, 包括appid, secret_id和secret_key + appid = 100000 # 替换为用户的appid + secret_id = u'xxxxxxxx' # 替换为用户的secret_id + secret_key = u'xxxxxxx' # 替换为用户的secret_key +   region = "cn-north"       # 替换为用户的region，目前可以为 cn-east/cn-south/cn-north/cn-southwest，分别对应于上海，广州，天津,西南园区 + config = CosConfig(Appid=appid, Region=region, Access_id=secret_id, Access_key=secret_key) #获取配置对象 + client = CosS3Client(config) #获取客户端对象 + + ############################################################################ + # 文件操作 # + ############################################################################ + # 1. 上传单个文件 + response = client.put_object( + Bucket='test01', + Body='TY'*1024*512*file_size, + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + + # 2. 下载单个文件 + response = client.get_object( + Bucket='test01', + Key=file_name, + ) + + # 3. 获取文件属性 + response = client.head_object( + Bucket='test01', + Key=file_name + ) + + # 4. 删除单个文件 + response = client.delete_object( + Bucket='test01', + Key=file_name + ) + + # 5. 创建分片上传 + response = client.create_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + ) + uploadid = get_id_from_xml(response.text) + + # 6. 删除分片上传 + response = client.abort_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + UploadId=uploadid + ) + + # 7. 再次创建分片上传 + response = client.create_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + ) + uploadid = response['UploadId'] + + # 8. 上传分片 + response = client.upload_part( + Bucket='test01', + Key='multipartfile.txt', + UploadId=uploadid, + PartNumber=1, + Body='A'*1024*1024*4 + ) + etag = response['ETag'] + + # 9. 列出分片 + response = clieent.list_parts( + Bucket='test01', + Key='mutilpartfile.txt', + UploadId=uploadid + ) + lst = response['Part'] + + # 10. 完成分片上传 + response = client.complete_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + UploadId=uploadid, + MultipartUpload={'Part': lst} + ) + + + ############################################################################ + # Bucket操作 # + ############################################################################ + # 1. 创建Bucket + response = client.create_bucket( + Bucket='test02', + ACL='public-read' + ) + + # 2. 删除Bucket + response = client.delete_bucket( + Bucket='test02' + ) + + # 3. 获取文件列表 + response = client.list_objects( + Bucket='test01' + ) diff --git a/qcloud_cos/__init__.py b/qcloud_cos/__init__.py index b193bef6..8472f4a1 100644 --- a/qcloud_cos/__init__.py +++ b/qcloud_cos/__init__.py @@ -1,3 +1,5 @@ -import cos_auth -import cos_threadpool -import cos_client +from .cos_client import CosS3Client +from .cos_client import CosConfig +from .cos_exception import CosServiceError +from .cos_exception import CosClientError +from .cos_auth import CosS3Auth diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 0f49b635..6cadc4e2 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- + import hmac import time import urllib diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 8916bf56..dd759b1f 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -1,11 +1,18 @@ # -*- coding=utf-8 -from cos_auth import CosS3Auth + import requests +import urllib import logging import sys import copy import xml.dom.minidom - +import xml.etree.ElementTree +from requests import Request, Session +from streambody import StreamBody +from xml2dict import Xml2Dict +from cos_auth import CosS3Auth +from cos_exception import CosClientError +from cos_exception import CosServiceError logging.basicConfig( level=logging.INFO, @@ -14,8 +21,8 @@ filename='cos_s3.log', filemode='w') logger = logging.getLogger(__name__) -fs_coding = sys.getfilesystemencoding() - +reload(sys) +sys.setdefaultencoding('utf-8') maplist = { 'ContentLength': 'Content-Length', 'ContentType': 'Content-Type', @@ -24,18 +31,12 @@ 'ContentDisposition': 'Content-Disposition', 'ContentEncoding': 'Content-Encoding', 'Expires': 'Expires', - 'Metadata': 'x-cos-meta- *', + 'Metadata': 'Metadata', 'ACL': 'x-cos-acl', 'GrantFullControl': 'x-cos-grant-full-control', 'GrantWrite': 'x-cos-grant-write', 'GrantRead': 'x-cos-grant-read', 'StorageClass': 'x-cos-storage-class', - 'PartNumber': 'partNumber', - 'UploadId': 'uploadId', - 'Delimiter': 'delimiter', - 'Marker': 'marker', - 'MaxKeys': 'max-keys', - 'Prefix': 'prefix', 'EncodingType': 'encoding-type' } @@ -44,31 +45,29 @@ def to_unicode(s): if isinstance(s, unicode): return s else: - return s.decode(fs_coding) + return s.decode('utf-8') def dict_to_xml(data): + """V5使用xml格式，将输入的dict转换为xml""" doc = xml.dom.minidom.Document() root = doc.createElement('CompleteMultipartUpload') doc.appendChild(root) - if 'Parts' not in data.keys(): - logger.error("Invalid Parameter, Parts Is Required!") - return '' + if 'Part' not in data.keys(): + raise CosClientError("Invalid Parameter, Part Is Required!") - for i in data['Parts']: + for i in data['Part']: nodePart = doc.createElement('Part') if 'PartNumber' not in i.keys(): - logger.error("Invalid Parameter, PartNumber Is Required!") - return '' + raise CosClientError("Invalid Parameter, PartNumber Is Required!") nodeNumber = doc.createElement('PartNumber') nodeNumber.appendChild(doc.createTextNode(str(i['PartNumber']))) if 'ETag' not in i.keys(): - logger.error("Invalid Parameter, ETag Is Required!") - return '' + raise CosClientError("Invalid Parameter, ETag Is Required!") nodeETag = doc.createElement('ETag') nodeETag.appendChild(doc.createTextNode(str(i['ETag']))) @@ -79,16 +78,35 @@ def dict_to_xml(data): return doc.toxml('utf-8') +def xml_to_dict(data): + """V5使用xml格式，将response中的xml转换为dict""" + root = xml.etree.ElementTree.fromstring(data) + xmldict = Xml2Dict(root) + return xmldict + + +def get_id_from_xml(data, name): + """解析xml中的特定字段""" + tree = xml.dom.minidom.parseString(data) + root = tree.documentElement + result = root.getElementsByTagName(name) + # use childNodes to get a list, if has no child get itself + return result[0].childNodes[0].nodeValue + + def mapped(headers): + """S3到COS参数的一个映射""" _headers = dict() for i in headers.keys(): if i in maplist: _headers[maplist[i]] = headers[i] + else: + raise CosClientError('No Parameter Named '+i+' Please Check It') return _headers class CosConfig(object): - + """config类，保存用户相关信息""" def __init__(self, Appid, Region, Access_id, Access_key): self._appid = Appid self._region = Region @@ -99,7 +117,10 @@ def __init__(self, Appid, Region, Access_id, Access_key): region=Region)) def uri(self, bucket, path=None): + """拼接url""" if path: + if path[0] == '/': + path = path[1:] url = u"http://{bucket}-{uid}.{region}.myqcloud.com/{path}".format( bucket=to_unicode(bucket), uid=self._appid, @@ -116,88 +137,137 @@ def uri(self, bucket, path=None): class CosS3Client(object): - - def __init__(self, conf, session=None): + """cos客户端类，封装相应请求""" + def __init__(self, conf, retry=2, session=None): self._conf = conf - self._upload_id = None - self._headers = [] - self._params = [] - self._md5 = [] - self._retry = 1 - self._file_num = 0 - self._folder_num = 0 - self._have_finished = 0 + self._retry = retry # 重试的次数，分片上传时可适当增大 if session is None: self._session = requests.session() else: self._session = session + def get_auth(self, Method, Bucket, Key=None, **kwargs): + """获取签名""" + headers = mapped(kwargs) + # TODO(tiedu) 检查header的参数合法性 + url = self._conf.uri(bucket=Bucket, path=Key) + r = Request(Method, url) + auth = CosS3Auth(self._conf._access_id, self._conf._access_key) + return auth(r).headers['Authorization'] + + def send_request(self, method, url, timeout=30, **kwargs): + try: + for j in range(self._retry): + if method == 'POST': + res = self._session.post(url, timeout=timeout, **kwargs) + elif method == 'GET': + res = self._session.get(url, timeout=timeout, **kwargs) + elif method == 'PUT': + res = self._session.put(url, timeout=timeout, **kwargs) + elif method == 'DELETE': + res = self._session.delete(url, timeout=timeout, **kwargs) + elif method == 'HEAD': + res = self._session.head(url, timeout=timeout, **kwargs) + if res.status_code < 300: + return res + except Exception as e: # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误 + logger.exception('url:%s, exception:%s' % (url, str(e))) + raise CosClientError(str(e)) + + if res.status_code >= 300: # 所有的3XX,4XX,5XX都认为是COSServiceError + msg = res.text + logger.error(msg) + raise CosServiceError(msg, res.status_code) + def put_object(self, Bucket, Body, Key, **kwargs): + """单文件上传接口，适用于小文件，最大不得超过5GB""" headers = mapped(kwargs) + if 'Metadata' in headers.keys(): + for i in headers['Metadata'].keys(): + headers[i] = headers['Metadata'][i] + headers.pop('Metadata') + url = self._conf.uri(bucket=Bucket, path=Key) logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) for j in range(self._retry): - rt = self._session.put( + rt = self.send_request( + method='PUT', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=Body, headers=headers) + if rt is None: + continue if rt.status_code == 200: break - logger.error(rt.headers) - return rt + logger.error(rt.text) + + response = dict() + response['ETag'] = rt.headers['ETag'] + return response def get_object(self, Bucket, Key, **kwargs): + """单文件下载接口""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key) logger.info("get object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.get( + rt = self.send_request( + method='GET', url=url, + stream=True, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + + response = dict() + response['Body'] = StreamBody(rt) + + for k in rt.headers.keys(): + response[k] = rt.headers[k] + return response + + def get_presigned_download_url(self, Bucket, Key): + """生成预签名的下载url""" + url = self._conf.uri(bucket=Bucket, path=Key) + sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key) + url = url + '?sign=' + urllib.quote(sign) + return url def delete_object(self, Bucket, Key, **kwargs): + """单文件删除接口""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key) logger.info("delete object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.delete( + rt = self.send_request( + method='DELETE', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 204: - break - logger.error(rt.headers) - return rt + return None def create_multipart_upload(self, Bucket, Key, **kwargs): + """创建分片上传，适用于大文件上传""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key+"?uploads") logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.post( + rt = self.send_request( + method='POST', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + + data = xml_to_dict(rt.text) + return data def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): + """上传分片，单个大小不得超过5GB""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key+"?partNumber={PartNumber}&uploadId={UploadId}".format( PartNumber=PartNumber, @@ -205,98 +275,104 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.put( + rt = self.send_request( + method='PUT', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=Body) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + response = dict() + response['ETag'] = rt.headers['ETag'] + return response def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, **kwargs): + """完成分片上传，组装后的文件不得小于1MB,否则会返回错误""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.post( + rt = self.send_request( + method='POST', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=dict_to_xml(MultipartUpload), + timeout=1200, # 分片上传大文件的时间比较长，设置为20min headers=headers) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + response = dict() + data = xml_to_dict(rt.text) + for key in data.keys(): + response[key[key.find('}')+1:]] = data[key] + return response def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): + """放弃一个已经存在的分片上传任务，删除所有已经存在的分片""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.delete( + rt = self.send_request( + method='DELETE', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + return None def list_parts(self, Bucket, Key, UploadId, **kwargs): + """列出已上传的分片""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.get( + rt = self.send_request( + method='GET', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + + data = xml_to_dict(rt.text) + if 'Part' in data.keys(): + if isinstance(data['Part'], list): + return data + else: # 只有一个part，将dict转为list，保持一致 + lst = [] + lst.append(data['Part']) + data['Part'] = lst + return data + else: + return data def create_bucket(self, Bucket, **kwargs): + """创建一个bucket""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("create bucket, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.put( + rt = self.send_request( + method='PUT', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + return None def delete_bucket(self, Bucket, **kwargs): + """删除一个bucket，bucket必须为空""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("delete bucket, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.delete( + rt = self.send_request( + method='DELETE', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 204: - break - logger.error(rt.headers) - return rt + return None - def list_objects(self, Bucket, Delimiter="", EncodingType="url", Marker="", MaxKeys=100, Prefix="", **kwargs): + def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", **kwargs): + """获取文件列表""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("list objects, url=:{url} ,headers=:{headers}".format( @@ -304,37 +380,83 @@ def list_objects(self, Bucket, Delimiter="", EncodingType="url", Marker="", MaxK headers=headers)) params = { 'delimiter': Delimiter, - 'encoding-type': EncodingType, 'marker': Marker, 'max-keys': MaxKeys, 'prefix': Prefix} - for j in range(self._retry): - rt = self._session.get( + rt = self.send_request( + method='GET', url=url, params=params, headers=headers, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + + data = xml_to_dict(rt.text) + if 'Contents' in data.keys(): + if isinstance(data['Contents'], list): + return data + else: # 只有一个Contents，将dict转为list，保持一致 + lst = [] + lst.append(data['Contents']) + data['Contents'] = lst + return data + else: + return data def head_object(self, Bucket, Key, **kwargs): + """获取文件信息""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key) - logger.info("put object, url=:{url} ,headers=:{headers}".format( + logger.info("head object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) - for j in range(self._retry): - rt = self._session.head( + rt = self.send_request( + method='HEAD', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 200: - break - logger.error(rt.headers) - return rt + return rt.headers + + def gen_copy_source_url(self, CopySource): + """拼接拷贝源url""" + if 'Bucket' in CopySource.keys(): + bucket = CopySource['Bucket'] + else: + raise CosClientError('CopySource Need Parameter Bucket') + if 'Key' in CopySource.keys(): + key = CopySource['Key'] + else: + raise CosClientError('CopySource Need Parameter Key') + url = self._conf.uri(bucket=bucket, path=key).encode('utf8') + url = url[7:] # copysource不支持http://开头，去除 + return url + def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): + """文件拷贝，文件信息修改""" + headers = mapped(kwargs) + headers['x-cos-copy-source'] = self.gen_copy_source_url(CopySource) + headers['x-cos-metadata-directive'] = CopyStatus + url = self._conf.uri(bucket=Bucket, path=Key) + logger.info("copy object, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + return data + + def list_buckets(self): + """列出所有bucket""" + url = 'http://service.cos.myqcloud.com/' + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + ) + data = xml_to_dict(rt.text) + return data if __name__ == "__main__": pass diff --git a/qcloud_cos/cos_exception.py b/qcloud_cos/cos_exception.py new file mode 100644 index 00000000..8610e3b3 --- /dev/null +++ b/qcloud_cos/cos_exception.py @@ -0,0 +1,86 @@ +# -*- coding=utf-8 + +import xml.dom.minidom + + +class CosException(Exception): + def __init__(self, message): + Exception.__init__(self, message) + + +def digest_xml(data): + msg = dict() + try: + tree = xml.dom.minidom.parseString(data) + root = tree.documentElement + + result = root.getElementsByTagName('Code') + msg['code'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('Message') + msg['message'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('Resource') + msg['resource'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('RequestId') + msg['requestid'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('TraceId') + msg['traceid'] = result[0].childNodes[0].nodeValue + return msg + except Exception as e: + return "Response Error Msg Is INVALID" + + +class CosClientError(CosException): + """Client端错误，如timeout""" + def __init__(self, message): + CosException.__init__(self, message) + + +class CosServiceError(CosException): + """COS Server端错误，可以获取特定的错误信息""" + def __init__(self, message, status_code): + CosException.__init__(self, message) + self._origin_msg = message + self._digest_msg = digest_xml(message) + self._status_code = status_code + + def get_origin_msg(self): + """获取原始的XML格式错误信息""" + return self._origin_msg + + def get_digest_msg(self): + """获取经过处理的dict格式的错误信息""" + return self._digest_msg + + def get_status_code(self): + """获取http error code""" + return self._status_code + + def get_error_code(self): + """获取COS定义的错误码描述,服务器返回错误信息格式出错时，返回空 """ + if isinstance(self._digest_msg, dict): + return self._digest_msg['code'] + return "Unknown" + + def get_error_msg(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['message'] + return "Unknown" + + def get_resource_location(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['resource'] + return "Unknown" + + def get_trace_id(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['requestid'] + return "Unknown" + + def get_request_id(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['traceid'] + return "Unknown" diff --git a/qcloud_cos/streambody.py b/qcloud_cos/streambody.py new file mode 100644 index 00000000..d28fbacc --- /dev/null +++ b/qcloud_cos/streambody.py @@ -0,0 +1,30 @@ +# -*- coding=utf-8 +import requests + + +class StreamBody(): + def __init__(self, rt): + self._rt = rt + + def get_raw_stream(self): + return self._rt.raw + + def get_stream(self, chunk_size=1024): + return self._rt.iter_content(chunk_size=chunk_size) + + def get_stream_to_file(self, file_name): + if 'Content-Length' in self._rt.headers: + content_len = int(self._rt.headers['Content-Length']) + else: + raise IOError("download failed without Content-Length header") + + file_len = 0 + with open(file_name, 'wb') as fp: + for chunk in self._rt.iter_content(chunk_size=1024): + if chunk: + file_len += len(chunk) + fp.write(chunk) + fp.flush() + fp.close() + if file_len != content_len: + raise IOError("download failed with incomplete file") diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py index 4728a5b3..8a1aa02c 100644 --- a/qcloud_cos/test.py +++ b/qcloud_cos/test.py @@ -1,12 +1,10 @@ # -*- coding=utf-8 -import cos_client import random import sys import os -import xml.dom.minidom -from xml.dom.minidom import parse from cos_client import CosS3Client from cos_client import CosConfig +from cos_exception import CosServiceError ACCESS_ID = os.environ["ACCESS_ID"] ACCESS_KEY = os.environ["ACCESS_KEY"] @@ -19,14 +17,6 @@ def gen_file(path, size): _file.close() -def get_id_from_xml(data): - tree = xml.dom.minidom.parseString(data) - root = tree.documentElement - result = root.getElementsByTagName('UploadId') - # use childNodes to get a list, if has no child get itself - return result[0].childNodes[0].nodeValue - - def setUp(): print "start test" @@ -42,114 +32,185 @@ def Test(): Access_id=ACCESS_ID, Access_key=ACCESS_KEY ) - client = cos_client.CosS3Client(conf) + client = CosS3Client(conf) - file_size = 2 + test_bucket = 'test01' + file_size = 2 # 方便CI通过 file_id = str(random.randint(0, 1000)) + str(random.randint(0, 1000)) file_name = "tmp" + file_id + "_" + str(file_size) + "MB" + print "Test Get Presigned Download URL " + url = client.get_presigned_download_url( + Bucket=test_bucket, + Key='test.txt' + ) + print url + + print "Test List Buckets" + response = client.list_buckets() + + copy_source = {'Bucket': 'test01', 'Key': '/test.txt'} + print "Test Copy Object From Other Bucket " + response = client.copy_object( + Bucket='test04', + Key='test.txt', + CopySource=copy_source + ) + + print "Test Put Object That Bucket Not Exist " + file_name + try: + response = client.put_object( + Bucket='test0xx', + Body='T'*1024*1024, + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + except CosServiceError as e: + print e.get_origin_msg() + print e.get_digest_msg() + print e.get_status_code() + print e.get_error_code() + print e.get_error_msg() + print e.get_resource_location() + print e.get_trace_id() + print e.get_request_id() + + special_file_name = "对象()*'/. 存![]^&*~储{|}~()" + print "Test Put Object Contains Special Characters " + special_file_name + response = client.put_object( + Bucket=test_bucket, + Body='S'*1024*1024, + Key=special_file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + + print "Test Get Object Contains Special Characters " + special_file_name + response = client.get_object( + Bucket=test_bucket, + Key=special_file_name, + ) + + print "Test Delete Object Contains Special Characters " + special_file_name + response = client.delete_object( + Bucket=test_bucket, + Key=special_file_name + ) + print "Test Put Object " + file_name + gen_file(file_name, file_size) + fp = open(file_name, 'rb') response = client.put_object( - Bucket='test01', - Body='TY'*1024*512*file_size, + Bucket=test_bucket, + Body=fp, Key=file_name, CacheControl='no-cache', ContentDisposition='download.txt', - ACL='public-read' + Metadata={ + "x-cos-meta-tiedu": "value1" + } ) - assert response.status_code == 200 + fp.close() + os.remove(file_name) print "Test Get Object " + file_name response = client.get_object( - Bucket='test01', + Bucket=test_bucket, Key=file_name, - ) - assert response.status_code == 200 + ) + # 返回一个raw stream + # fp = response['Body'].get_raw_stream() + # 返回一个generator + # stream_generator = response['Body'].get_stream(stream_size=1024*512) + response['Body'].get_stream_to_file('cos.txt') + if os.path.exists('cos.txt'): + os.remove('cos.txt') print "Test Head Object " + file_name response = client.head_object( - Bucket='test01', + Bucket=test_bucket, Key=file_name ) - assert response.status_code == 200 print "Test Delete Object " + file_name - response = client.delete_object( - Bucket='test01', + response = client.head_object( + Bucket=test_bucket, Key=file_name ) - assert response.status_code == 204 print "Test List Objects" response = client.list_objects( - Bucket='test01' + Bucket='test04' ) - assert response.status_code == 200 + print response print "Test Create Bucket" response = client.create_bucket( - Bucket='test02', + Bucket='test'+file_id, ACL='public-read' ) - assert response.status_code == 200 print "Test Delete Bucket" response = client.delete_bucket( - Bucket='test02' + Bucket='test'+file_id ) - assert response.status_code == 204 print "Test Create MultipartUpload" response = client.create_multipart_upload( - Bucket='test01', + Bucket=test_bucket, Key='multipartfile.txt', ) - assert response.status_code == 200 - uploadid = get_id_from_xml(response.text) + uploadid = response['UploadId'] print "Test Abort MultipartUpload" response = client.abort_multipart_upload( - Bucket='test01', + Bucket=test_bucket, Key='multipartfile.txt', UploadId=uploadid ) - assert response.status_code == 200 print "Test Create MultipartUpload" response = client.create_multipart_upload( - Bucket='test01', + Bucket=test_bucket, Key='multipartfile.txt', ) - uploadid = get_id_from_xml(response.text) - assert response.status_code == 200 + uploadid = response['UploadId'] - print "Test Upload Part" + print "Test Upload Part1" response = client.upload_part( - Bucket='test01', + Bucket=test_bucket, Key='multipartfile.txt', UploadId=uploadid, PartNumber=1, - Body='A'*1024*1024*4 + Body='A'*1024*1024*2 + ) + + print "Test Upload Part2" + response = client.upload_part( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid, + PartNumber=2, + Body='B'*1024*1024*2 ) - etag = response.headers['ETag'] - assert response.status_code == 200 print "List Upload Parts" response = client.list_parts( - Bucket='test01', + Bucket=test_bucket, Key='multipartfile.txt', UploadId=uploadid ) - assert response.status_code == 200 + print response + lst = response['Part'] print "Test Complete MultipartUpload" response = client.complete_multipart_upload( - Bucket='test01', + Bucket=test_bucket, Key='multipartfile.txt', UploadId=uploadid, - MultipartUpload={'Parts': [{'PartNumber': 1, 'ETag': etag}]} + MultipartUpload={'Part': lst} ) - assert response.status_code == 200 if __name__ == "__main__": setUp() diff --git a/qcloud_cos/xml2dict.py b/qcloud_cos/xml2dict.py new file mode 100644 index 00000000..cfa23414 --- /dev/null +++ b/qcloud_cos/xml2dict.py @@ -0,0 +1,46 @@ +# -*- coding=utf-8 +import xml.etree.ElementTree + + +class Xml2Dict(dict): + def __init__(self, parent_node): + if parent_node.items(): + self.updateDict(dict(parent_node.items())) + for element in parent_node: + if len(element): + aDict = Xml2Dict(element) + self.updateDict({element.tag: aDict}) + elif element.items(): + elementattrib = element.items() + if element.text: + elementattrib.append((element.tag, element.text)) + self.updateDict({element.tag: dict(elementattrib)}) + else: + self.updateDict({element.tag: element.text}) + + def updateDict(self, aDict): + for key in aDict.keys(): + if key in self: + value = self.pop(key) + if type(value) is not list: + lst = list() + lst.append(value) + lst.append(aDict[key]) + self.update({key: lst}) + else: + value.append(aDict[key]) + self.update({key: value}) + else: + self.update({key: aDict[key]}) + +if __name__ == "__main__": + s = """ + + 10 + 1test1 + 2test2 + 3test3 + """ + root = xml.etree.ElementTree.fromstring(s) + xmldict = XmlDictConfig(root) + print xmldict diff --git a/setup.py b/setup.py index a7bbc943..042d9e74 100644 --- a/setup.py +++ b/setup.py @@ -21,14 +21,13 @@ def long_description(): setup( name='cos-python-sdk-v5', - version='0.0.1', + version='1.0.0', url='https://www.qcloud.com/', license='MIT', - author='lewzylu', - author_email='327874225@qq.com', + author='tiedu, lewzylu, channingliu', + author_email='dutie123@qq.com', description='cos-python-sdk-v5', long_description=long_description(), packages=find_packages(), install_requires=requirements() - } )