diff --git a/.travis.yml b/.travis.yml index a51222c3..0deba634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,15 @@ -sudo: false -language: python - -python: - - "2.6" - - "2.7" - -install: - - pip install requests - - pip install nose - - pip install pep8 - - pip install tox - -script: - - pep8 --max-line-length=180 - - nosetests -s - - tox +sudo: false +language: python + +python: + - "2.6" + - "2.7" + +install: + - pip install requests + - pip install nose + - pip install pep8 + +script: + - pep8 --max-line-length=180 qcloud_cos/. + - nosetests -s -v diff --git a/README.md b/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..62b9a075 --- /dev/null +++ b/README.rst @@ -0,0 +1,131 @@ +Qcloud COSv5 SDK +####################### + +介绍 +_______ + +腾讯云COSV5Python SDK, 目前可以支持Python2.6与Python2.7。 + +安装指南 +__________ + +使用pip安装 :: + + pip install -U cos-python-sdk-v5 + + +手动安装:: + + python setup.py install + +使用方法 +__________ + +使用python sdk，参照https://github.com/tencentyun/cos-python-sdk-v5/blob/master/qcloud_cos/test.py + +.. code:: python + + # 设置用户属性, 包括appid, secret_id和secret_key + appid = 100000 # 替换为用户的appid + secret_id = u'xxxxxxxx' # 替换为用户的secret_id + secret_key = u'xxxxxxx' # 替换为用户的secret_key +   region = "cn-north"       # 替换为用户的region，目前可以为 cn-east/cn-south/cn-north/cn-southwest，分别对应于上海，广州，天津,西南园区 + config = CosConfig(Appid=appid, Region=region, Access_id=secret_id, Access_key=secret_key) #获取配置对象 + client = CosS3Client(config) #获取客户端对象 + + ############################################################################ + # 文件操作 # + ############################################################################ + # 1. 上传单个文件 + response = client.put_object( + Bucket='test01', + Body='TY'*1024*512*file_size, + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + + # 2. 下载单个文件 + response = client.get_object( + Bucket='test01', + Key=file_name, + ) + + # 3. 获取文件属性 + response = client.head_object( + Bucket='test01', + Key=file_name + ) + + # 4. 删除单个文件 + response = client.delete_object( + Bucket='test01', + Key=file_name + ) + + # 5. 创建分片上传 + response = client.create_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + ) + uploadid = get_id_from_xml(response.text) + + # 6. 删除分片上传 + response = client.abort_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + UploadId=uploadid + ) + + # 7. 再次创建分片上传 + response = client.create_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + ) + uploadid = response['UploadId'] + + # 8. 上传分片 + response = client.upload_part( + Bucket='test01', + Key='multipartfile.txt', + UploadId=uploadid, + PartNumber=1, + Body='A'*1024*1024*4 + ) + etag = response['ETag'] + + # 9. 列出分片 + response = clieent.list_parts( + Bucket='test01', + Key='mutilpartfile.txt', + UploadId=uploadid + ) + lst = response['Part'] + + # 10. 完成分片上传 + response = client.complete_multipart_upload( + Bucket='test01', + Key='multipartfile.txt', + UploadId=uploadid, + MultipartUpload={'Part': lst} + ) + + + ############################################################################ + # Bucket操作 # + ############################################################################ + # 1. 创建Bucket + response = client.create_bucket( + Bucket='test02', + ACL='public-read' + ) + + # 2. 删除Bucket + response = client.delete_bucket( + Bucket='test02' + ) + + # 3. 获取文件列表 + response = client.list_objects( + Bucket='test01' + ) diff --git a/qcloud_cos/__init__.py b/qcloud_cos/__init__.py index b193bef6..8472f4a1 100644 --- a/qcloud_cos/__init__.py +++ b/qcloud_cos/__init__.py @@ -1,3 +1,5 @@ -import cos_auth -import cos_threadpool -import cos_client +from .cos_client import CosS3Client +from .cos_client import CosConfig +from .cos_exception import CosServiceError +from .cos_exception import CosClientError +from .cos_auth import CosS3Auth diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index 6635cbb3..6cadc4e2 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -1,16 +1,16 @@ # -*- coding: utf-8 -*- + import hmac import time import urllib import hashlib import logging -import requests from urllib import quote from urlparse import urlparse from requests.auth import AuthBase logger = logging.getLogger(__name__) -#fix a bug which can't send header + class CosS3Auth(AuthBase): def __init__(self, access_id, secret_key, expire=10000): @@ -71,5 +71,4 @@ def __call__(self, r): if __name__ == "__main__": - url = 'http://lewzylu01-1252448703.cn-south.myqcloud.com/a.txt' - \ No newline at end of file + pass diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 955182b8..d3dc79af 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -1,93 +1,139 @@ # -*- coding=utf-8 -from cos_auth import CosS3Auth -from cos_threadpool import SimpleThreadPool -import time + import requests -from os import path -from contextlib import closing -from xml.dom import minidom +import urllib import logging import sys -import os import copy - +import xml.dom.minidom +import xml.etree.ElementTree +from requests import Request, Session +from streambody import StreamBody +from xml2dict import Xml2Dict +from cos_auth import CosS3Auth +from cos_exception import CosClientError +from cos_exception import CosServiceError + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S', + filename='cos_s3.log', + filemode='w') logger = logging.getLogger(__name__) -fs_coding = sys.getfilesystemencoding() - -maplist = {'CacheControl':'Cache-Control', - 'ContentDisposition':'Content-Disposition', - 'ContentEncoding':'Content-Encoding', - 'Expires':'Expires', - 'Metadata':'x-cos-meta- *', - 'ACL':'x-cos-acl', - 'GrantFullControl':'x-cos-grant-full-control', - 'GrantWrite':'x-cos-grant-write', - 'GrantRead':'x-cos-grant-read', - 'StorageClass':'x-cos-storage-class', +reload(sys) +sys.setdefaultencoding('utf-8') +maplist = { + 'ContentLength': 'Content-Length', + 'ContentType': 'Content-Type', + 'ContentMD5': 'Content-MD5', + 'CacheControl': 'Cache-Control', + 'ContentDisposition': 'Content-Disposition', + 'ContentEncoding': 'Content-Encoding', + 'Expires': 'Expires', + 'Metadata': 'Metadata', + 'ACL': 'x-cos-acl', + 'GrantFullControl': 'x-cos-grant-full-control', + 'GrantWrite': 'x-cos-grant-write', + 'GrantRead': 'x-cos-grant-read', + 'StorageClass': 'x-cos-storage-class', + 'EncodingType': 'encoding-type' } + def to_unicode(s): if isinstance(s, unicode): return s else: - return s.decode(fs_coding) + return s.decode('utf-8') -def to_printable_str(s): - if isinstance(s, unicode): - return s.encode(fs_coding) - else: - return s +def dict_to_xml(data): + """V5使用xml格式，将输入的dict转换为xml""" + doc = xml.dom.minidom.Document() + root = doc.createElement('CompleteMultipartUpload') + doc.appendChild(root) + + if 'Part' not in data.keys(): + raise CosClientError("Invalid Parameter, Part Is Required!") + + for i in data['Part']: + nodePart = doc.createElement('Part') + + if 'PartNumber' not in i.keys(): + raise CosClientError("Invalid Parameter, PartNumber Is Required!") + + nodeNumber = doc.createElement('PartNumber') + nodeNumber.appendChild(doc.createTextNode(str(i['PartNumber']))) + + if 'ETag' not in i.keys(): + raise CosClientError("Invalid Parameter, ETag Is Required!") + + nodeETag = doc.createElement('ETag') + nodeETag.appendChild(doc.createTextNode(str(i['ETag']))) + + nodePart.appendChild(nodeNumber) + nodePart.appendChild(nodeETag) + root.appendChild(nodePart) + return doc.toxml('utf-8') + +def xml_to_dict(data): + """V5使用xml格式，将response中的xml转换为dict""" + root = xml.etree.ElementTree.fromstring(data) + xmldict = Xml2Dict(root) + xmlstr = str(xmldict) + xmlstr = xmlstr.replace("{http://www.qcloud.com/document/product/436/7751}", "") + xmlstr = xmlstr.replace("{http://www.w3.org/2001/XMLSchema-instance}", "") + xmldict = eval(xmlstr) + return xmldict -def view_bar(num, total): - ret = 1.0*num / total - ag = ret * 100 - ab = "\r [%-50s]%.2f%%" % ('='*int(ret*50), ag, ) - sys.stdout.write(ab) - sys.stdout.flush() +def get_id_from_xml(data, name): + """解析xml中的特定字段""" + tree = xml.dom.minidom.parseString(data) + root = tree.documentElement + result = root.getElementsByTagName(name) + # use childNodes to get a list, if has no child get itself + return result[0].childNodes[0].nodeValue -def getTagText(root, tag): - node = root.getElementsByTagName(tag)[0] - rc = "" - for node in node.childNodes: - if node.nodeType in (node.TEXT_NODE, node.CDATA_SECTION_NODE): - rc = rc + node.data def mapped(headers): - _headers = copy.copy(headers) + """S3到COS参数的一个映射""" + _headers = dict() for i in headers.keys(): if i in maplist: - del _headers[i] _headers[maplist[i]] = headers[i] + else: + raise CosClientError('No Parameter Named '+i+' Please Check It') return _headers -class CosConfig(object): - def __init__(self, appid, region, access_id, access_key, part_size=1, max_thread=5, *args, **kwargs): - self._appid = appid - self._region = region - self._access_id = access_id - self._access_key = access_key - self._part_size = min(10, part_size) - self._max_thread = min(10, max_thread) - logger.info("config parameter-> appid: {appid}, region: {region}, part_size: {part_size}, max_thread: {max_thread}".format( - appid=appid, - region=region, - part_size=part_size, - max_thread=max_thread)) +class CosConfig(object): + """config类，保存用户相关信息""" + def __init__(self, Appid, Region, Access_id, Access_key, Token=None): + self._appid = Appid + self._region = Region + self._access_id = Access_id + self._access_key = Access_key + self._token = Token + logger.info("config parameter-> appid: {appid}, region: {region}".format( + appid=Appid, + region=Region)) def uri(self, bucket, path=None): + """拼接url""" if path: - url = u"http://{bucket}-{uid}.{region}.myqcloud.com/{path}".format( + if path[0] == '/': + path = path[1:] + url = u"http://{bucket}-{uid}.cos.{region}.myqcloud.com/{path}".format( bucket=to_unicode(bucket), uid=self._appid, region=self._region, path=to_unicode(path) ) else: - url = u"http://{bucket}-{uid}.{region}.myqcloud.com".format( + url = u"http://{bucket}-{uid}.cos.{region}.myqcloud.com".format( bucket=to_unicode(bucket), uid=self._appid, region=self._region @@ -95,99 +141,408 @@ def uri(self, bucket, path=None): return url -class ObjectInterface(object): - - def __init__(self, conf, session=None): +class CosS3Client(object): + """cos客户端类，封装相应请求""" + def __init__(self, conf, retry=1, session=None): self._conf = conf - self._upload_id = None - self._headers = [] - self._params = [] - self._md5 = [] - self._retry = 2 - self._file_num = 0 - self._folder_num = 0 - self._have_finished = 0 + self._retry = retry # 重试的次数，分片上传时可适当增大 if session is None: self._session = requests.session() else: self._session = session - + def get_auth(self, Method, Bucket, Key=None, Expired=10000, headers={}, params={}): + """获取签名""" + url = self._conf.uri(bucket=Bucket, path=Key) + r = Request(Method, url, headers=headers, params=params) + auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Expired) + return auth(r).headers['Authorization'] + + def send_request(self, method, url, timeout=30, **kwargs): + if self._conf._token is not None: + kwargs['headers']['x-cos-security-token'] = self._conf._token + try: + for j in range(self._retry): + if method == 'POST': + res = self._session.post(url, timeout=timeout, **kwargs) + elif method == 'GET': + res = self._session.get(url, timeout=timeout, **kwargs) + elif method == 'PUT': + res = self._session.put(url, timeout=timeout, **kwargs) + elif method == 'DELETE': + res = self._session.delete(url, timeout=timeout, **kwargs) + elif method == 'HEAD': + res = self._session.head(url, timeout=timeout, **kwargs) + if res.status_code < 300: + return res + except Exception as e: # 捕获requests抛出的如timeout等客户端错误,转化为客户端错误 + logger.exception('url:%s, exception:%s' % (url, str(e))) + raise CosClientError(str(e)) + + if res.status_code >= 400: # 所有的4XX,5XX都认为是COSServiceError + if method == 'HEAD' and res.status_code == 404: # Head 需要处理 + info = dict() + info['code'] = 'NoSuchResource' + info['message'] = 'The Resource You Head Not Exist' + info['resource'] = url + info['requestid'] = res.headers['x-cos-request-id'] + info['traceid'] = res.headers['x-cos-trace-id'] + logger.error(info) + raise CosServiceError(method, info, res.status_code) + else: + msg = res.text + if msg == '': # 服务器没有返回Error Body时 给出头部的信息 + msg = res.headers + logger.error(msg) + raise CosServiceError(method, msg, res.status_code) + + # s3 object interface begin def put_object(self, Bucket, Body, Key, **kwargs): + """单文件上传接口，适用于小文件，最大不得超过5GB""" headers = mapped(kwargs) + if 'Metadata' in headers.keys(): + for i in headers['Metadata'].keys(): + headers[i] = headers['Metadata'][i] + headers.pop('Metadata') + url = self._conf.uri(bucket=Bucket, path=Key) - for j in range(self._retry): - rt = self._session.put(url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=Body, headers=headers) - if rt.status_code == 200: - break - return rt + logger.info("put object, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + data=Body, + headers=headers) + + response = dict() + response['ETag'] = rt.headers['ETag'] + return response def get_object(self, Bucket, Key, **kwargs): + """单文件下载接口""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key) - for j in range(self._retry): - rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key),headers=headers) - if rt.status_code == 200: - break - return rt + logger.info("get object, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + stream=True, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + + response = dict() + response['Body'] = StreamBody(rt) + + for k in rt.headers.keys(): + response[k] = rt.headers[k] + return response + + def get_presigned_download_url(self, Bucket, Key, Expired=10000): + """生成预签名的下载url""" + url = self._conf.uri(bucket=Bucket, path=Key) + sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key, Expired=10000) + url = url + '?sign=' + urllib.quote(sign) + return url def delete_object(self, Bucket, Key, **kwargs): + """单文件删除接口""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key) - for j in range(self._retry): - rt = self._session.delete(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key),headers=headers) - if rt.status_code == 204: - break - return rt - - - -class BucketInterface(object): - - def __init__(self, conf, session=None): - self._conf = conf - self._upload_id = None - self._md5 = [] - self._have_finished = 0 - self._retry = 2 - if session is None: - self._session = requests.session() + logger.info("delete object, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='DELETE', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def head_object(self, Bucket, Key, **kwargs): + """获取文件信息""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key) + logger.info("head object, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='HEAD', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return rt.headers + + def gen_copy_source_url(self, CopySource): + """拼接拷贝源url""" + if 'Bucket' in CopySource.keys(): + bucket = CopySource['Bucket'] else: - self._session = session + raise CosClientError('CopySource Need Parameter Bucket') + if 'Key' in CopySource.keys(): + key = CopySource['Key'] + else: + raise CosClientError('CopySource Need Parameter Key') + url = self._conf.uri(bucket=bucket, path=key).encode('utf8') + url = url[7:] # copysource不支持http://开头，去除 + return url - def put_bucket(self, Bucket, **kwargs): + def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): + """文件拷贝，文件信息修改""" + headers = mapped(kwargs) + headers['x-cos-copy-source'] = self.gen_copy_source_url(CopySource) + headers['x-cos-metadata-directive'] = CopyStatus + url = self._conf.uri(bucket=Bucket, path=Key) + logger.info("copy object, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + return data + + def create_multipart_upload(self, Bucket, Key, **kwargs): + """创建分片上传，适用于大文件上传""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key+"?uploads") + logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='POST', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + + data = xml_to_dict(rt.text) + return data + + def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): + """上传分片，单个大小不得超过5GB""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key+"?partNumber={PartNumber}&uploadId={UploadId}".format( + PartNumber=PartNumber, + UploadId=UploadId)) + logger.info("put object, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + data=Body) + response = dict() + response['ETag'] = rt.headers['ETag'] + return response + + def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, **kwargs): + """完成分片上传，组装后的文件不得小于1MB,否则会返回错误""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) + logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='POST', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + data=dict_to_xml(MultipartUpload), + timeout=1200, # 分片上传大文件的时间比较长，设置为20min + headers=headers) + data = xml_to_dict(rt.text) + return data + + def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): + """放弃一个已经存在的分片上传任务，删除所有已经存在的分片""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) + logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='DELETE', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def list_parts(self, Bucket, Key, UploadId, **kwargs): + """列出已上传的分片""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) + logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + if 'Part' in data.keys() and isinstance(data['Part'], dict): # 只有一个part，将dict转为list，保持一致 + lst = [] + lst.append(data['Part']) + data['Part'] = lst + return data + + def put_object_acl(self, Bucket, Key, **kwargs): + """设置object ACL""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key+"?acl") + logger.info("put object acl, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def get_object_acl(self, Bucket, Key, **kwargs): + """获取object ACL""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path=Key+"?acl") + logger.info("get object acl, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict): + lst = [] + lst.append(data['AccessControlList']['Grant']) + data['AccessControlList']['Grant'] = lst + return data + + # s3 bucket interface begin + def create_bucket(self, Bucket, **kwargs): + """创建一个bucket""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) - for j in range(self._retry): - rt = self._session.put(url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) - if rt.status_code == 200: - break - return rt + logger.info("create bucket, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None def delete_bucket(self, Bucket, **kwargs): + """删除一个bucket，bucket必须为空""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) - for j in range(self._retry): - rt = self._session.delete(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key),headers=headers) - if rt.status_code == 204: - break - return rt - - -class CosS3Client(object): - - def __init__(self, conf): - self._conf = conf - self._session = requests.session() - - def obj_int(self, local_path='', cos_path=''): - return ObjectInterface(conf=self._conf, session=self._session) - - def buc_int(self): - return BucketInterface(conf=self._conf, session=self._session) - + logger.info("delete bucket, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='DELETE', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", **kwargs): + """获取文件列表""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket) + logger.info("list objects, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + params = { + 'delimiter': Delimiter, + 'marker': Marker, + 'max-keys': MaxKeys, + 'prefix': Prefix} + rt = self.send_request( + method='GET', + url=url, + params=params, + headers=headers, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) + + data = xml_to_dict(rt.text) + if 'Contents' in data.keys() and isinstance(data['Contents'], dict): # 只有一个Contents，将dict转为list，保持一致 + lst = [] + lst.append(data['Contents']) + data['Contents'] = lst + return data + + def head_bucket(self, Bucket, **kwargs): + """获取bucket信息""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket) + logger.info("head bucket, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='HEAD', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def put_bucket_acl(self, Bucket, **kwargs): + """设置bucket ACL""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?acl") + logger.info("put bucket acl, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='PUT', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + return None + + def get_bucket_acl(self, Bucket, **kwargs): + """获取bucket ACL""" + headers = mapped(kwargs) + url = self._conf.uri(bucket=Bucket, path="?acl") + logger.info("get bucket acl, url=:{url} ,headers=:{headers}".format( + url=url, + headers=headers)) + rt = self.send_request( + method='GET', + url=url, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + headers=headers) + data = xml_to_dict(rt.text) + if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict): + lst = [] + lst.append(data['AccessControlList']['Grant']) + data['AccessControlList']['Grant'] = lst + return data + + # service interface begin + def list_buckets(self, **kwargs): + """列出所有bucket""" + headers = mapped(kwargs) + url = 'http://service.cos.myqcloud.com/' + rt = self.send_request( + method='GET', + url=url, + headers=headers, + auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + ) + data = xml_to_dict(rt.text) + if data['Buckets'] is not None and isinstance(data['Buckets']['Bucket'], dict): + lst = [] + lst.append(data['Buckets']['Bucket']) + data['Buckets']['Bucket'] = lst + return data if __name__ == "__main__": pass diff --git a/qcloud_cos/cos_exception.py b/qcloud_cos/cos_exception.py new file mode 100644 index 00000000..bb8c2dea --- /dev/null +++ b/qcloud_cos/cos_exception.py @@ -0,0 +1,90 @@ +# -*- coding=utf-8 + +import xml.dom.minidom + + +class CosException(Exception): + def __init__(self, message): + Exception.__init__(self, message) + + +def digest_xml(data): + msg = dict() + try: + tree = xml.dom.minidom.parseString(data) + root = tree.documentElement + + result = root.getElementsByTagName('Code') + msg['code'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('Message') + msg['message'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('Resource') + msg['resource'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('RequestId') + msg['requestid'] = result[0].childNodes[0].nodeValue + + result = root.getElementsByTagName('TraceId') + msg['traceid'] = result[0].childNodes[0].nodeValue + return msg + except Exception as e: + return "Response Error Msg Is INVALID" + + +class CosClientError(CosException): + """Client端错误，如timeout""" + def __init__(self, message): + CosException.__init__(self, message) + + +class CosServiceError(CosException): + """COS Server端错误，可以获取特定的错误信息""" + def __init__(self, method, message, status_code): + CosException.__init__(self, message) + if method == 'HEAD': # 对HEAD进行特殊处理 + self._origin_msg = '' + self._digest_msg = message + else: + self._origin_msg = message + self._digest_msg = digest_xml(message) + self._status_code = status_code + + def get_origin_msg(self): + """获取原始的XML格式错误信息""" + return self._origin_msg + + def get_digest_msg(self): + """获取经过处理的dict格式的错误信息""" + return self._digest_msg + + def get_status_code(self): + """获取http error code""" + return self._status_code + + def get_error_code(self): + """获取COS定义的错误码描述,服务器返回错误信息格式出错时，返回空 """ + if isinstance(self._digest_msg, dict): + return self._digest_msg['code'] + return "Unknown" + + def get_error_msg(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['message'] + return "Unknown" + + def get_resource_location(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['resource'] + return "Unknown" + + def get_trace_id(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['requestid'] + return "Unknown" + + def get_request_id(self): + if isinstance(self._digest_msg, dict): + return self._digest_msg['traceid'] + return "Unknown" diff --git a/qcloud_cos/streambody.py b/qcloud_cos/streambody.py new file mode 100644 index 00000000..d28fbacc --- /dev/null +++ b/qcloud_cos/streambody.py @@ -0,0 +1,30 @@ +# -*- coding=utf-8 +import requests + + +class StreamBody(): + def __init__(self, rt): + self._rt = rt + + def get_raw_stream(self): + return self._rt.raw + + def get_stream(self, chunk_size=1024): + return self._rt.iter_content(chunk_size=chunk_size) + + def get_stream_to_file(self, file_name): + if 'Content-Length' in self._rt.headers: + content_len = int(self._rt.headers['Content-Length']) + else: + raise IOError("download failed without Content-Length header") + + file_len = 0 + with open(file_name, 'wb') as fp: + for chunk in self._rt.iter_content(chunk_size=1024): + if chunk: + file_len += len(chunk) + fp.write(chunk) + fp.flush() + fp.close() + if file_len != content_len: + raise IOError("download failed with incomplete file") diff --git a/qcloud_cos/test.py b/qcloud_cos/test.py new file mode 100644 index 00000000..077291de --- /dev/null +++ b/qcloud_cos/test.py @@ -0,0 +1,264 @@ +# -*- coding=utf-8 +import random +import sys +import os +from cos_client import CosS3Client +from cos_client import CosConfig +from cos_exception import CosServiceError + +ACCESS_ID = os.environ["ACCESS_ID"] +ACCESS_KEY = os.environ["ACCESS_KEY"] + + +def gen_file(path, size): + _file = open(path, 'w') + _file.seek(1024*1024*size) + _file.write('cos') + _file.close() + + +def print_error_msg(e): + print e.get_origin_msg() + print e.get_digest_msg() + print e.get_status_code() + print e.get_error_code() + print e.get_error_msg() + print e.get_resource_location() + print e.get_trace_id() + print e.get_request_id() + + +def setUp(): + print "start test" + + +def tearDown(): + print "function teardown" + + +def Test(): + conf = CosConfig( + Appid="1252448703", + Region="cn-north", + Access_id=ACCESS_ID, + Access_key=ACCESS_KEY + ) + client = CosS3Client(conf) + + test_bucket = 'test01' + file_size = 2 # 方便CI通过 + file_id = str(random.randint(0, 1000)) + str(random.randint(0, 1000)) + file_name = "tmp" + file_id + "_" + str(file_size) + "MB" + + print "Test Get Presigned Download URL " + url = client.get_presigned_download_url( + Bucket=test_bucket, + Key='test.txt' + ) + print url + + print "Test List Buckets" + response = client.list_buckets() + + copy_source = {'Bucket': 'test01', 'Key': '/test.txt'} + print "Test Copy Object From Other Bucket " + + response = client.copy_object( + Bucket='test04', + Key='test.txt', + CopySource=copy_source + ) + + print "Test Put Object That Bucket Not Exist " + file_name + try: + response = client.put_object( + Bucket='test0xx', + Body='T'*1024*1024, + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + except CosServiceError as e: + print_error_msg(e) + + special_file_name = "对象()*'/. 存![]^&*~储{|}~()" + print "Test Put Object Contains Special Characters " + special_file_name + response = client.put_object( + Bucket=test_bucket, + Body='S'*1024*1024, + Key=special_file_name, + CacheControl='no-cache', + ContentDisposition='download.txt' + ) + + print "Test Get Object Contains Special Characters " + special_file_name + response = client.get_object( + Bucket=test_bucket, + Key=special_file_name, + ) + + print "Test Delete Object Contains Special Characters " + special_file_name + response = client.delete_object( + Bucket=test_bucket, + Key=special_file_name + ) + + print "Test Put Object " + file_name + gen_file(file_name, file_size) + fp = open(file_name, 'rb') + response = client.put_object( + Bucket=test_bucket, + Body=fp, + Key=file_name, + CacheControl='no-cache', + ContentDisposition='download.txt', + Metadata={ + "x-cos-meta-tiedu": "value1" + } + ) + fp.close() + os.remove(file_name) + + print "Test Get Object " + file_name + response = client.get_object( + Bucket=test_bucket, + Key=file_name, + ) + # 返回一个raw stream + # fp = response['Body'].get_raw_stream() + # 返回一个generator + # stream_generator = response['Body'].get_stream(stream_size=1024*512) + response['Body'].get_stream_to_file('cos.txt') + if os.path.exists('cos.txt'): + os.remove('cos.txt') + + print "Test Head Object " + file_name + response = client.head_object( + Bucket=test_bucket, + Key=file_name + ) + + print "Test Head Object " + file_name + "123" + try: + response = client.head_object( + Bucket=test_bucket, + Key=file_name+"123" + ) + except CosServiceError as e: + print_error_msg(e) + + print "Test Put Object ACL " + file_name + response = client.put_object_acl( + Bucket=test_bucket, + Key=file_name, + ACL='public-read-write' + ) + + print "Test Get Object ACL" + file_name + response = client.get_object_acl( + Bucket=test_bucket, + Key=file_name + ) + + print "Test Delete Object " + file_name + response = client.head_object( + Bucket=test_bucket, + Key=file_name + ) + + print "Test List Objects" + response = client.list_objects( + Bucket=test_bucket + ) + + print "Test Create Bucket" + response = client.create_bucket( + Bucket='test'+file_id, + ACL='public-read' + ) + + print "Test PUT Bucket ACL" + try: + response = client.put_bucket_acl( + Bucket='test'+file_id, + ACL='public-read-writea' + ) + except CosServiceError as e: + print_error_msg(e) + + print "Test GET Bucket ACL" + response = client.get_bucket_acl( + Bucket='test'+file_id, + ) + + print "Test Delete Bucket" + response = client.delete_bucket( + Bucket='test'+file_id + ) + + print "Test Head Bucket" + try: + response = client.head_bucket( + Bucket='test'+file_id + ) + except CosServiceError as e: + print_error_msg(e) + + print "Test Create MultipartUpload" + response = client.create_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + ) + uploadid = response['UploadId'] + + print "Test Abort MultipartUpload" + response = client.abort_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid + ) + + print "Test Create MultipartUpload" + response = client.create_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + ) + uploadid = response['UploadId'] + + print "Test Upload Part1" + response = client.upload_part( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid, + PartNumber=1, + Body='A'*1024*1024*2 + ) + + print "Test Upload Part2" + response = client.upload_part( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid, + PartNumber=2, + Body='B'*1024*1024*2 + ) + + print "List Upload Parts" + response = client.list_parts( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid + ) + lst = response['Part'] + + print "Test Complete MultipartUpload" + response = client.complete_multipart_upload( + Bucket=test_bucket, + Key='multipartfile.txt', + UploadId=uploadid, + MultipartUpload={'Part': lst} + ) + +if __name__ == "__main__": + setUp() + Test() diff --git a/qcloud_cos/xml2dict.py b/qcloud_cos/xml2dict.py new file mode 100644 index 00000000..dedd5c14 --- /dev/null +++ b/qcloud_cos/xml2dict.py @@ -0,0 +1,46 @@ +# -*- coding=utf-8 +import xml.etree.ElementTree + + +class Xml2Dict(dict): + def __init__(self, parent_node): + if parent_node.items(): + self.updateDict(dict(parent_node.items())) + for element in parent_node: + if len(element): + aDict = Xml2Dict(element) + self.updateDict({element.tag: aDict}) + elif element.items(): + elementattrib = element.items() + if element.text: + elementattrib.append((element.tag, element.text)) + self.updateDict({element.tag: dict(elementattrib)}) + else: + self.updateDict({element.tag: element.text}) + + def updateDict(self, aDict): + for key in aDict.keys(): + if key in self: + value = self.pop(key) + if type(value) is not list: + lst = list() + lst.append(value) + lst.append(aDict[key]) + self.update({key: lst}) + else: + value.append(aDict[key]) + self.update({key: value}) + else: + self.update({key: aDict[key]}) + +if __name__ == "__main__": + s = """ + + 10 + 1test1 + 2test2 + 3test3 + """ + root = xml.etree.ElementTree.fromstring(s) + xmldict = Xml2Dict(root) + print xmldict diff --git a/setup.py b/setup.py index a7bbc943..042d9e74 100644 --- a/setup.py +++ b/setup.py @@ -21,14 +21,13 @@ def long_description(): setup( name='cos-python-sdk-v5', - version='0.0.1', + version='1.0.0', url='https://www.qcloud.com/', license='MIT', - author='lewzylu', - author_email='327874225@qq.com', + author='tiedu, lewzylu, channingliu', + author_email='dutie123@qq.com', description='cos-python-sdk-v5', long_description=long_description(), packages=find_packages(), install_requires=requirements() - } ) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 611174cd..00000000 --- a/tox.ini +++ /dev/null @@ -1,15 +0,0 @@ -[tox] -envlist = py26,py27 -[testenv] -deps= - requests - lxml - -commands= - coscmd -h - coscmd config -h - coscmd upload -h - coscmd config -a AKID52nXBd1vnTxHPOuwSmtobGixzpSstB1f -s ay9MVSGsIagct1SmygHgtuiLm14vc9Uh -u 1252448703 -b sdktestgz -r cn-south - # dd if=/dev/urandom of=10m.txt bs=1M count=10 - # coscmd -v upload 10m.txt upload/10m.txt -