Skip to content

Commit

Permalink
Support all special characters
Browse files Browse the repository at this point in the history
  • Loading branch information
@dt3310321
dt3310321 committed Oct 11, 2017
1 parent fc608b4 commit 9222932
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 45 deletions.
29 changes: 14 additions & 15 deletions qcloud_cos/cos_auth.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,29 +22,28 @@ def filter_headers(data):

class CosS3Auth(AuthBase):

def __init__(self, access_id, secret_key, expire=10000):
def __init__(self, access_id, secret_key, key='', params={}, expire=10000):
self._access_id = access_id
self._secret_key = secret_key
self._expire = expire

def __call__(self, r):
method = r.method.lower() # 获取小写method
uri = urllib.unquote(r.url)
rt = urlparse(uri) # 解析host以及params
logger.debug("url parse: " + str(rt))
if rt.query != "" and ("&" in rt.query or '=' in rt.query):
uri_params = dict(map(lambda s: s.lower().split('='), rt.query.split('&')))
uri_params = {}
elif rt.query != "":
uri_params = {rt.query: ""}
self._params = params
if key:
if key[0] == '/':
self._path = key
else:
self._path = '/' + key
else:
uri_params = {}
self._path = '/'

def __call__(self, r):
path = self._path
uri_params = self._params
headers = filter_headers(r.headers)
# reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+)
headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode
format_str = "{method}\n{host}\n{params}\n{headers}\n".format(
method=method.lower(),
host=rt.path,
method=r.method.lower(),
host=path,
params=urllib.urlencode(sorted(uri_params.items())),
headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items())))
)
Expand Down
61 changes: 31 additions & 30 deletions qcloud_cos/cos_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import xml.dom.minidom
import xml.etree.ElementTree
from requests import Request, Session
from urllib import quote
from streambody import StreamBody
from xml2dict import Xml2Dict
from dicttoxml import dicttoxml
Expand Down Expand Up @@ -193,7 +194,7 @@ def uri(self, bucket, path=None):
path=to_unicode(path)
)
else:
url = u"http://{bucket}-{uid}.{region}.myqcloud.com".format(
url = u"http://{bucket}-{uid}.{region}.myqcloud.com/".format(
bucket=to_unicode(bucket),
uid=self._appid,
region=self._region
Expand All @@ -211,11 +212,11 @@ def __init__(self, conf, retry=1, session=None):
else:
self._session = session

def get_auth(self, Method, Bucket, Key=None, Expired=300, headers={}, params={}):
def get_auth(self, Method, Bucket, Key='', Expired=300, headers={}, params={}):
"""获取签名"""
url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~'))
r = Request(Method, url, headers=headers, params=params)
auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Expired)
auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Key, params, Expired)
return auth(r).headers['Authorization']

def send_request(self, method, url, timeout=30, **kwargs):
Expand Down Expand Up @@ -266,14 +267,14 @@ def put_object(self, Bucket, Body, Key, **kwargs):
headers[i] = headers['Metadata'][i]
headers.pop('Metadata')

url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) # 提前对key做encode
logger.info("put object, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='PUT',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
data=Body,
headers=headers)

Expand All @@ -284,15 +285,15 @@ def put_object(self, Bucket, Body, Key, **kwargs):
def get_object(self, Bucket, Key, **kwargs):
"""单文件下载接口"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~'))
logger.info("get object, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='GET',
url=url,
stream=True,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)

response = dict()
Expand All @@ -304,36 +305,36 @@ def get_object(self, Bucket, Key, **kwargs):

def get_presigned_download_url(self, Bucket, Key, Expired=300):
"""生成预签名的下载url"""
url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~'))
sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key, Expired=300)
url = urllib.quote(url.encode('utf8'), ':/') + '?sign=' + urllib.quote(sign)
url = url + '?sign=' + urllib.quote(sign)
return url

def delete_object(self, Bucket, Key, **kwargs):
"""单文件删除接口"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~'))
logger.info("delete object, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='DELETE',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)
return None

def head_object(self, Bucket, Key, **kwargs):
"""获取文件信息"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~'))
logger.info("head object, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='HEAD',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)
return rt.headers

Expand Down Expand Up @@ -373,14 +374,14 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs):
if CopyStatus != 'Copy' and CopyStatus != 'Replaced':
raise CosClientError('CopyStatus must be Copy or Replaced')
headers['x-cos-metadata-directive'] = CopyStatus
url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~'))
logger.info("copy object, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='PUT',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)
data = xml_to_dict(rt.text)
return data
Expand All @@ -393,14 +394,14 @@ def create_multipart_upload(self, Bucket, Key, **kwargs):
headers[i] = headers['Metadata'][i]
headers.pop('Metadata')

url = self._conf.uri(bucket=Bucket, path=Key+"?uploads")
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploads")
logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='POST',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)

data = xml_to_dict(rt.text)
Expand All @@ -409,7 +410,7 @@ def create_multipart_upload(self, Bucket, Key, **kwargs):
def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs):
"""上传分片，单个大小不得超过5GB"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key+"?partNumber={PartNumber}&uploadId={UploadId}".format(
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?partNumber={PartNumber}&uploadId={UploadId}".format(
PartNumber=PartNumber,
UploadId=UploadId))
logger.info("put object, url=:{url} ,headers=:{headers}".format(
Expand All @@ -419,7 +420,7 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs):
method='PUT',
url=url,
headers=headers,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
data=Body)
response = dict()
response['ETag'] = rt.headers['ETag']
Expand All @@ -428,14 +429,14 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs):
def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, **kwargs):
"""完成分片上传，组装后的文件不得小于1MB,否则会返回错误"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId))
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId))
logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='POST',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
data=dict_to_xml(MultipartUpload),
timeout=1200, # 分片上传大文件的时间比较长，设置为20min
headers=headers)
Expand All @@ -445,14 +446,14 @@ def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, *
def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs):
"""放弃一个已经存在的分片上传任务，删除所有已经存在的分片"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId))
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId))
logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='DELETE',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)
return None

Expand All @@ -465,14 +466,14 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='url', MaxParts=1000, P
'max-parts': MaxParts,
'encoding-type': EncodingType}

url = self._conf.uri(bucket=Bucket, path=Key)
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~'))
logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='GET',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers,
params=params)
data = xml_to_dict(rt.text)
Expand All @@ -491,29 +492,29 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs):
if AccessControlPolicy:
xml_config = format_xml(data=AccessControlPolicy, root='AccessControlPolicy', lst=lst)
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key+"?acl")
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl")
logger.info("put object acl, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='PUT',
url=url,
data=xml_config,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)
return None

def get_object_acl(self, Bucket, Key, **kwargs):
"""获取object ACL"""
headers = mapped(kwargs)
url = self._conf.uri(bucket=Bucket, path=Key+"?acl")
url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl")
logger.info("get object acl, url=:{url} ,headers=:{headers}".format(
url=url,
headers=headers))
rt = self.send_request(
method='GET',
url=url,
auth=CosS3Auth(self._conf._access_id, self._conf._access_key),
auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key),
headers=headers)
data = xml_to_dict(rt.text, "type", "Type")
if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict):
Expand Down

0 comments on commit 9222932

Please sign in to comment.