diff --git a/qcloud_cos/cos_auth.py b/qcloud_cos/cos_auth.py index e976156c..dc0fe3e7 100644 --- a/qcloud_cos/cos_auth.py +++ b/qcloud_cos/cos_auth.py @@ -22,29 +22,28 @@ def filter_headers(data): class CosS3Auth(AuthBase): - def __init__(self, access_id, secret_key, expire=10000): + def __init__(self, access_id, secret_key, key='', params={}, expire=10000): self._access_id = access_id self._secret_key = secret_key self._expire = expire - - def __call__(self, r): - method = r.method.lower() # 获取小写method - uri = urllib.unquote(r.url) - rt = urlparse(uri) # 解析host以及params - logger.debug("url parse: " + str(rt)) - if rt.query != "" and ("&" in rt.query or '=' in rt.query): - uri_params = dict(map(lambda s: s.lower().split('='), rt.query.split('&'))) - uri_params = {} - elif rt.query != "": - uri_params = {rt.query: ""} + self._params = params + if key: + if key[0] == '/': + self._path = key + else: + self._path = '/' + key else: - uri_params = {} + self._path = '/' + + def __call__(self, r): + path = self._path + uri_params = self._params headers = filter_headers(r.headers) # reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+) headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode format_str = "{method}\n{host}\n{params}\n{headers}\n".format( - method=method.lower(), - host=rt.path, + method=r.method.lower(), + host=path, params=urllib.urlencode(sorted(uri_params.items())), headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items()))) ) diff --git a/qcloud_cos/cos_client.py b/qcloud_cos/cos_client.py index 5ba41da1..b41557d3 100644 --- a/qcloud_cos/cos_client.py +++ b/qcloud_cos/cos_client.py @@ -11,6 +11,7 @@ import xml.dom.minidom import xml.etree.ElementTree from requests import Request, Session +from urllib import quote from streambody import StreamBody from xml2dict import Xml2Dict from dicttoxml import dicttoxml @@ -193,7 +194,7 @@ def uri(self, bucket, path=None): path=to_unicode(path) ) else: - url = u"http://{bucket}-{uid}.{region}.myqcloud.com".format( + url = u"http://{bucket}-{uid}.{region}.myqcloud.com/".format( bucket=to_unicode(bucket), uid=self._appid, region=self._region @@ -211,11 +212,11 @@ def __init__(self, conf, retry=1, session=None): else: self._session = session - def get_auth(self, Method, Bucket, Key=None, Expired=300, headers={}, params={}): + def get_auth(self, Method, Bucket, Key='', Expired=300, headers={}, params={}): """获取签名""" - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) r = Request(Method, url, headers=headers, params=params) - auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Expired) + auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Key, params, Expired) return auth(r).headers['Authorization'] def send_request(self, method, url, timeout=30, **kwargs): @@ -266,14 +267,14 @@ def put_object(self, Bucket, Body, Key, **kwargs): headers[i] = headers['Metadata'][i] headers.pop('Metadata') - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) # 提前对key做encode logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='PUT', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), data=Body, headers=headers) @@ -284,7 +285,7 @@ def put_object(self, Bucket, Body, Key, **kwargs): def get_object(self, Bucket, Key, **kwargs): """单文件下载接口""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("get object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -292,7 +293,7 @@ def get_object(self, Bucket, Key, **kwargs): method='GET', url=url, stream=True, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) response = dict() @@ -304,36 +305,36 @@ def get_object(self, Bucket, Key, **kwargs): def get_presigned_download_url(self, Bucket, Key, Expired=300): """生成预签名的下载url""" - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) sign = self.get_auth(Method='GET', Bucket=Bucket, Key=Key, Expired=300) - url = urllib.quote(url.encode('utf8'), ':/') + '?sign=' + urllib.quote(sign) + url = url + '?sign=' + urllib.quote(sign) return url def delete_object(self, Bucket, Key, **kwargs): """单文件删除接口""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("delete object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return None def head_object(self, Bucket, Key, **kwargs): """获取文件信息""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("head object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='HEAD', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return rt.headers @@ -373,14 +374,14 @@ def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): if CopyStatus != 'Copy' and CopyStatus != 'Replaced': raise CosClientError('CopyStatus must be Copy or Replaced') headers['x-cos-metadata-directive'] = CopyStatus - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("copy object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='PUT', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) data = xml_to_dict(rt.text) return data @@ -393,14 +394,14 @@ def create_multipart_upload(self, Bucket, Key, **kwargs): headers[i] = headers['Metadata'][i] headers.pop('Metadata') - url = self._conf.uri(bucket=Bucket, path=Key+"?uploads") + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploads") logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) data = xml_to_dict(rt.text) @@ -409,7 +410,7 @@ def create_multipart_upload(self, Bucket, Key, **kwargs): def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): """上传分片，单个大小不得超过5GB""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?partNumber={PartNumber}&uploadId={UploadId}".format( + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?partNumber={PartNumber}&uploadId={UploadId}".format( PartNumber=PartNumber, UploadId=UploadId)) logger.info("put object, url=:{url} ,headers=:{headers}".format( @@ -419,7 +420,7 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): method='PUT', url=url, headers=headers, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), data=Body) response = dict() response['ETag'] = rt.headers['ETag'] @@ -428,14 +429,14 @@ def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, **kwargs): """完成分片上传，组装后的文件不得小于1MB,否则会返回错误""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), data=dict_to_xml(MultipartUpload), timeout=1200, # 分片上传大文件的时间比较长，设置为20min headers=headers) @@ -445,14 +446,14 @@ def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, * def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): """放弃一个已经存在的分片上传任务，删除所有已经存在的分片""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?uploadId={UploadId}".format(UploadId=UploadId)) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return None @@ -465,14 +466,14 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='url', MaxParts=1000, P 'max-parts': MaxParts, 'encoding-type': EncodingType} - url = self._conf.uri(bucket=Bucket, path=Key) + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')) logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers, params=params) data = xml_to_dict(rt.text) @@ -491,7 +492,7 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): if AccessControlPolicy: xml_config = format_xml(data=AccessControlPolicy, root='AccessControlPolicy', lst=lst) headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?acl") + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl") logger.info("put object acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) @@ -499,21 +500,21 @@ def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs): method='PUT', url=url, data=xml_config, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) return None def get_object_acl(self, Bucket, Key, **kwargs): """获取object ACL""" headers = mapped(kwargs) - url = self._conf.uri(bucket=Bucket, path=Key+"?acl") + url = self._conf.uri(bucket=Bucket, path=quote(Key, '/-_.~')+"?acl") logger.info("get object acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, - auth=CosS3Auth(self._conf._access_id, self._conf._access_key), + auth=CosS3Auth(self._conf._access_id, self._conf._access_key, Key), headers=headers) data = xml_to_dict(rt.text, "type", "Type") if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict):