Skip to content

Commit

Permalink
Merge pull request #15 from dt3310321/s3
Browse files Browse the repository at this point in the history 
S3 兼容
  • Loading branch information
@dt3310321
dt3310321 authored Sep 19, 2017
2 parents 9ec5a98 + 8e6200f commit 1b8a6e5
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 20 deletions.
16 changes: 10 additions & 6 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,19 @@ __________

使用python sdk，参照https://github.com/tencentyun/cos-python-sdk-v5/blob/master/qcloud_cos/test.py

cos最新可用地域,参照https://www.qcloud.com/document/product/436/6224

.. code:: python
# 设置用户属性, 包括appid, secret_id和secret_key
# 设置用户属性, 包括appid, secret_id, secret_key, region
appid = 100000 # 替换为用户的appid
secret_id = u'xxxxxxxx' # 替换为用户的secret_id
secret_key = u'xxxxxxx' # 替换为用户的secret_key
  region = "cn-north"       # 替换为用户的region，目前可以为 cn-east/cn-south/cn-north/cn-southwest，分别对应于上海，广州，天津,西南园区
   config = CosConfig(Appid=appid, Region=region, Access_id=secret_id, Access_key=secret_key, Token='')  #获取配置对象 使用临时秘钥需要传入Token，默认为空
   client = CosS3Client(config)                                                               #获取客户端对象
  region = "ap-beiging-1"    # 替换为用户的region
token = '' # 使用临时秘钥需要传入Token，默认为空,可不填
config = CosConfig(Appid=appid, Region=region, Access_id=secret_id, Access_key=secret_key, Token=token) #获取配置对象
client = CosS3Client(config) #获取客户端对象
############################################################################
# 文件操作 #
Expand Down Expand Up @@ -99,14 +103,14 @@ __________
Key='mutilpartfile.txt',
UploadId=uploadid
)
lst = response['Part']
lst = response['Part'] # list_parts最大数量为1000
# 10. 完成分片上传
response = client.complete_multipart_upload(
Bucket='test01',
Key='multipartfile.txt',
UploadId=uploadid,
MultipartUpload={'Part': lst}
MultipartUpload={'Part': lst} # 超过1000个分块，请本地保存分块信息，再complete
)
Expand Down
75 changes: 64 additions & 11 deletions qcloud_cos/cos_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,31 @@
sys.setdefaultencoding('utf-8')
maplist = {
'ContentLength': 'Content-Length',
'ContentType': 'Content-Type',
'ContentMD5': 'Content-MD5',
'ContentType': 'Content-Type',
'CacheControl': 'Cache-Control',
'ContentDisposition': 'Content-Disposition',
'ContentEncoding': 'Content-Encoding',
'ContentLanguage': 'Content-Language',
'Expires': 'Expires',
'ResponseContentType': 'response-content-type',
'ResponseContentLanguage': 'response-content-language',
'ResponseExpires': 'response-expires',
'ResponseCacheControl': 'response-cache-control',
'ResponseContentDisposition': 'response-content-disposition',
'ResponseContentEncoding': 'response-content-encoding',
'Metadata': 'Metadata',
'ACL': 'x-cos-acl',
'GrantFullControl': 'x-cos-grant-full-control',
'GrantWrite': 'x-cos-grant-write',
'GrantRead': 'x-cos-grant-read',
'StorageClass': 'x-cos-storage-class',
'EncodingType': 'encoding-type'
'Range': 'Range',
'IfMatch': 'If-Match',
'IfNoneMatch': 'If-None-Match',
'IfModifiedSince': 'If-Modified-Since',
'IfUnmodifiedSince': 'If-Unmodified-Since',
'VersionId': 'x-cos-version-id',
}


Expand Down Expand Up @@ -109,11 +121,39 @@ def mapped(headers):
return _headers


def format_region(region):
if region.find('cos.') != -1:
return region # 传入cos.ap-beijing-1这样显示加上cos.的region
if region == 'cn-north' or region == 'cn-south' or region == 'cn-east' or region == 'cn-south-2' or region == 'cn-southwest' or region == 'sg':
return region # 老域名不能加cos.
# 支持v4域名映射到v5
if region == 'cossh':
return 'cos.ap-shanghai'
if region == 'cosgz':
return 'cos.ap-guangzhou'
if region == 'cosbj':
return 'cos.ap-beijing'
if region == 'costj':
return 'cos.ap-beijing-1'
if region == 'coscd':
return 'cos.ap-chengdu'
if region == 'cossgp':
return 'cos.ap-singapore'
if region == 'coshk':
return 'cos.ap-hongkong'
if region == 'cosca':
return 'cos.na-toronto'
if region == 'cosger':
return 'cos.eu-frankfurt'

return 'cos.' + region # 新域名加上cos.


class CosConfig(object):
"""config类，保存用户相关信息"""
def __init__(self, Appid, Region, Access_id, Access_key, Token=None):
self._appid = Appid
self._region = Region
self._region = format_region(Region)
self._access_id = Access_id
self._access_key = Access_key
self._token = Token
Expand All @@ -126,14 +166,14 @@ def uri(self, bucket, path=None):
if path:
if path[0] == '/':
path = path[1:]
url = u"http://{bucket}-{uid}.cos.{region}.myqcloud.com/{path}".format(
url = u"http://{bucket}-{uid}.{region}.myqcloud.com/{path}".format(
bucket=to_unicode(bucket),
uid=self._appid,
region=self._region,
path=to_unicode(path)
)
else:
url = u"http://{bucket}-{uid}.cos.{region}.myqcloud.com".format(
url = u"http://{bucket}-{uid}.{region}.myqcloud.com".format(
bucket=to_unicode(bucket),
uid=self._appid,
region=self._region
Expand Down Expand Up @@ -246,7 +286,7 @@ def get_presigned_download_url(self, Bucket, Key, Expired=300):
"""生成预签名的下载url"""
url = self._conf.uri(bucket=Bucket, path=Key)
sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key, Expired=300)
url = url + '?sign=' + urllib.quote(sign)
url = urllib.quote(url.encode('utf8'), ':/') + '?sign=' + urllib.quote(sign)
return url

def delete_object(self, Bucket, Key, **kwargs):
Expand Down Expand Up @@ -311,6 +351,11 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs):
def create_multipart_upload(self, Bucket, Key, **kwargs):
"""创建分片上传，适用于大文件上传"""
headers = mapped(kwargs)
if 'Metadata' in headers.keys():
for i in headers['Metadata'].keys():
headers[i] = headers['Metadata'][i]
headers.pop('Metadata')

url = self._conf.uri(bucket=Bucket, path=Key+"?uploads")
logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format(
url=url,
Expand Down Expand Up @@ -374,18 +419,25 @@ def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs):
headers=headers)
return None

def list_parts(self, Bucket, Key, UploadId, **kwargs):
def list_parts(self, Bucket, Key, UploadId, EncodingType='url', MaxParts=1000, PartNumberMarker=0, **kwargs):
"""列出已上传的分片"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId))
params = {
'uploadId': UploadId,
'part-number-marker': PartNumberMarker,
'max-parts': MaxParts,
'encoding-type': EncodingType}

url = self._conf.uri(bucket=Bucket, path=Key)
logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='GET',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
headers=headers)
headers=headers,
params=params)
data = xml_to_dict(rt.text)
if 'Part' in data.keys() and isinstance(data['Part'], dict): # 只有一个part，将dict转为list，保持一致
lst = []
Expand Down Expand Up @@ -455,7 +507,7 @@ def delete_bucket(self, Bucket, **kwargs):
headers=headers)
return None

def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", **kwargs):
def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", EncodingType="url", **kwargs):
"""获取文件列表"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket)
Expand All @@ -466,7 +518,8 @@ def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="",
'delimiter': Delimiter,
'marker': Marker,
'max-keys': MaxKeys,
'prefix': Prefix}
'prefix': Prefix,
'encoding-type': EncodingType}
rt = self.send_request(
method='GET',
url=url,
Expand Down
6 changes: 3 additions & 3 deletions qcloud_cos/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def tearDown():
def Test():
conf = CosConfig(
Appid="1252448703",
Region="cn-north",
Region="ap-beijing-1",
Access_id=ACCESS_ID,
Access_key=ACCESS_KEY
)
Expand All @@ -53,7 +53,7 @@ def Test():
print "Test Get Presigned Download URL "
url = client.get_presigned_download_url(
Bucket=test_bucket,
Key='test.txt'
Key='中文.txt'
)
print url

Expand Down Expand Up @@ -81,7 +81,7 @@ def Test():
except CosServiceError as e:
print_error_msg(e)

special_file_name = "对象()*'/. 存![]^&*~储{|}~()"
special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
print "Test Put Object Contains Special Characters " + special_file_name
response = client.put_object(
Bucket=test_bucket,
Expand Down

0 comments on commit 1b8a6e5

Please sign in to comment.