Skip to content

Commit

Permalink
Merge pull request tencentyun#48 from dt3310321/s3
Browse files Browse the repository at this point in the history 
S3
  • Loading branch information
@dt3310321
dt3310321 authored May 4, 2018
2 parents f8d36b5 + 37e3789 commit 0be14bc
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 40 deletions.
3 changes: 2 additions & 1 deletion qcloud_cos/cos_auth.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,13 @@ def __call__(self, r):
path = self._path
uri_params = self._params
headers = filter_headers(r.headers)
uri_params = dict([(k.lower(), v.lower()) for k, v in uri_params.items()])
# reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+)
headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode
format_str = "{method}\n{host}\n{params}\n{headers}\n".format(
method=r.method.lower(),
host=path,
params=urllib.urlencode(sorted(uri_params.items())),
params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20'),
headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items())))
)
logger.debug("format str: " + format_str)
Expand Down
46 changes: 31 additions & 15 deletions qcloud_cos/cos_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def send_request(self, method, url, timeout=30, **kwargs):
timeout = self._conf._timeout
if self._conf._token is not None:
kwargs['headers']['x-cos-security-token'] = self._conf._token
kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.1'
kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.2'
try:
for j in range(self._retry):
if method == 'POST':
Expand Down Expand Up @@ -391,7 +391,7 @@ def delete_objects(self, Bucket, Delete={}, **kwargs):
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
headers=headers)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Deleted', 'Error'])
format_dict(data, ['Deleted', 'Error'])
return data

def head_object(self, Bucket, Key, **kwargs):
Expand Down Expand Up @@ -723,9 +723,9 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, Part
headers=headers,
params=params)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Part'])
format_dict(data, ['Part'])
if decodeflag:
data = decode_result(data, ['Key'], [])
decode_result(data, ['Key'], [])
return data

def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs):
Expand Down Expand Up @@ -938,9 +938,9 @@ def list_objects(self, Bucket, Prefix="", Delimiter="", Marker="", MaxKeys=1000,
headers=headers,
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
data = xml_to_dict(rt.text)
data = format_dict(data, ['Contents', 'CommonPrefixes'])
format_dict(data, ['Contents', 'CommonPrefixes'])
if decodeflag:
data = decode_result(
decode_result(
data,
[
'Prefix',
Expand Down Expand Up @@ -1006,9 +1006,9 @@ def list_objects_versions(self, Bucket, Prefix="", Delimiter="", KeyMarker="", V
headers=headers,
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
data = xml_to_dict(rt.text)
data = format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes'])
format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes'])
if decodeflag:
data = decode_result(
decode_result(
data,
[
'Prefix',
Expand Down Expand Up @@ -1078,9 +1078,9 @@ def list_multipart_uploads(self, Bucket, Prefix="", Delimiter="", KeyMarker="",
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))

data = xml_to_dict(rt.text)
data = format_dict(data, ['Upload', 'CommonPrefixes'])
format_dict(data, ['Upload', 'CommonPrefixes'])
if decodeflag:
data = decode_result(
decode_result(
data,
[
'Prefix',
Expand Down Expand Up @@ -1349,7 +1349,17 @@ def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs):
LifecycleConfiguration=lifecycle_config
)
"""
lst = ['<Rule>', '<Tag>', '</Tag>', '</Rule>'] # 类型为list的标签
# 类型为list的标签
lst = [
'<Rule>',
'<Tag>',
'<Transition>',
'<NoncurrentVersionTransition>',
'</NoncurrentVersionTransition>',
'</Transition>',
'</Tag>',
'</Rule>'
]
xml_config = format_xml(data=LifecycleConfiguration, root='LifecycleConfiguration', lst=lst)
headers = mapped(kwargs)
headers['Content-MD5'] = get_md5(xml_config)
Expand Down Expand Up @@ -1393,7 +1403,12 @@ def get_bucket_lifecycle(self, Bucket, **kwargs):
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
headers=headers)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Rule'])
format_dict(data, ['Rule'])
if 'Rule' in data.keys():
for rule in data['Rule']:
format_dict(rule, ['Transition', 'NoncurrentVersionTransition'])
if 'Filter' in rule.keys():
format_dict(rule['Filter'], ['Tag'])
return data

def delete_bucket_lifecycle(self, Bucket, **kwargs):
Expand Down Expand Up @@ -1596,7 +1611,7 @@ def get_bucket_replication(self, Bucket, **kwargs):
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
headers=headers)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Rule'])
format_dict(data, ['Rule'])
return data

def delete_bucket_replication(self, Bucket, **kwargs):
Expand Down Expand Up @@ -1814,11 +1829,12 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num,
UploadId=uploadid,
PartNumberMarker=part_number_marker
)
parts_info.extend(response['Part'])
if 'Part' in response:
parts_info.extend(response['Part'])
if response['IsTruncated'] == 'false':
list_over_status = True
else:
part_number_marker = int(response['NextMarker'])
part_number_marker = int(response['NextPartNumberMarker'])
for part in parts_info:
part_num = int(part['PartNumber'])
# 如果分块数量大于本地计算出的最大数量,校验失败
Expand Down
97 changes: 73 additions & 24 deletions ut/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

SECRET_ID = os.environ["SECRET_ID"]
SECRET_KEY = os.environ["SECRET_KEY"]
test_bucket = "test01-1252448703"
TRAVIS_FLAG = os.environ["TRAVIS_FLAG"]
test_bucket = 'cos-python-v5-test-' + str(sys.version_info[0]) + '-' + str(sys.version_info[1]) + '-' + '1252448703'
test_object = "test.txt"
special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
conf = CosConfig(
Expand All @@ -23,6 +24,19 @@
client = CosS3Client(conf)


def _create_test_bucket(test_bucket):
try:
response = client.create_bucket(
Bucket=test_bucket,
)
except Exception as e:
if e.get_error_code() == 'BucketAlreadyOwnedByYou':
print('BucketAlreadyOwnedByYou')
else:
raise e
return None


def get_raw_md5(data):
m2 = hashlib.md5(data)
etag = '"' + str(m2.hexdigest()) + '"'
Expand All @@ -37,22 +51,24 @@ def gen_file(path, size):


def print_error_msg(e):
print e.get_origin_msg()
print e.get_digest_msg()
print e.get_status_code()
print e.get_error_code()
print e.get_error_msg()
print e.get_resource_location()
print e.get_trace_id()
print e.get_request_id()
print (e.get_origin_msg())
print (e.get_digest_msg())
print (e.get_status_code())
print (e.get_error_code())
print (e.get_error_msg())
print (e.get_resource_location())
print (e.get_trace_id())
print (e.get_request_id())


def setUp():
print "start test..."
print ("start test...")
print ("start create bucket " + test_bucket)
_create_test_bucket(test_bucket)


def tearDown():
print "function teardown"
print ("function teardown")


def test_put_get_delete_object_10MB():
Expand Down Expand Up @@ -146,6 +162,11 @@ def test_put_object_non_exist_bucket():

def test_put_object_acl():
"""设置object acl"""
response = client.put_object(
Bucket=test_bucket,
Key=test_object,
Body='test acl'
)
response = client.put_object_acl(
Bucket=test_bucket,
Key=test_object,
Expand All @@ -160,6 +181,10 @@ def test_get_object_acl():
Key=test_object
)
assert response
response = client.delete_object(
Bucket=test_bucket,
Key=test_object
)


def test_copy_object_diff_bucket():
Expand Down Expand Up @@ -352,7 +377,7 @@ def test_get_bucket_acl_normal():
def test_list_objects():
"""列出bucket下的objects"""
response = client.list_objects(
Bucket=test_bucket,
Bucket='test01-1252448703',
MaxKeys=100,
Prefix='中文',
Delimiter='/'
Expand All @@ -376,7 +401,7 @@ def test_get_presigned_url():
Key='中文.txt'
)
assert url
print url
print (url)


def test_get_bucket_location():
Expand Down Expand Up @@ -430,10 +455,27 @@ def test_put_get_delete_lifecycle():
lifecycle_config = {
'Rule': [
{
'Expiration': {'Date': get_date(2030, 5, 1)},
'ID': '123',
'Filter': {'Prefix': ''},
'Status': 'Enabled',
'Filter': {
# 作用于带标签键 datalevel 和值 backup 的标签的对象
'Tag': [
{
'Key': 'datalevel',
'Value': 'backup'
}
]
},
'Transation': [
{
# 30天后转换为Standard_IA
'Days': 30,
'StorageClass': 'Standard_IA'
}
],
'Expiration': {
# 3650天后过期删除
'Days': 3650
}
}
]
}
Expand Down Expand Up @@ -479,7 +521,7 @@ def test_put_get_delete_replication():
{
'ID': '123',
'Status': 'Enabled',
'Prefix': 'replication',
'Prefix': '中文',
'Destination': {
'Bucket': 'qcs:id/0:cos:cn-south:appid/1252448703:replicationsouth'
}
Expand All @@ -499,7 +541,7 @@ def test_put_get_delete_replication():
Bucket=test_bucket
)
assert response
# delete lifecycle
# delete replication
response = client.delete_bucket_replication(
Bucket=test_bucket
)
Expand Down Expand Up @@ -547,7 +589,10 @@ def test_list_multipart_uploads():
def test_upload_file_multithreading():
"""根据文件大小自动选择分块大小,多线程并发上传提高上传速度"""
file_name = "thread_1GB"
gen_file(file_name, 5) # set 5MB beacuse travis too slow
file_size = 1024
if TRAVIS_FLAG == 'true':
file_size = 5 # set 5MB beacuse travis too slow
gen_file(file_name, file_size)
st = time.time() # 记录开始时间
response = client.upload_file(
Bucket=test_bucket,
Expand All @@ -560,7 +605,7 @@ def test_upload_file_multithreading():
ed = time.time() # 记录结束时间
if os.path.exists(file_name):
os.remove(file_name)
print ed - st
print (ed - st)


def test_copy_file_automatically():
Expand Down Expand Up @@ -608,7 +653,8 @@ def test_use_get_auth():
Key='test.txt',
Params={'acl': '', 'unsed': '123'}
)
response = requests.get('http://test01-1252448703.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123', headers={'Authorization': auth})
url = 'http://' + test_bucket + '.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123'
response = requests.get(url, headers={'Authorization': auth})
assert response.status_code == 200


Expand Down Expand Up @@ -646,16 +692,15 @@ def test_put_get_bucket_logging():
response = logging_client.get_bucket_logging(
Bucket=logging_bucket
)
print response
print (response)
assert response['LoggingEnabled']['TargetBucket'] == logging_bucket
assert response['LoggingEnabled']['TargetPrefix'] == 'test'


def test_put_object_enable_md5():
"""上传文件,SDK计算content-md5头部"""
file_size = 10
file_name = 'test_object_sdk_caculate_md5.file'
gen_file(file_name, 10)
gen_file(file_name, 1)
with open(file_name, 'rb') as f:
etag = get_raw_md5(f.read())
with open(file_name, 'rb') as fp:
Expand Down Expand Up @@ -686,6 +731,10 @@ def test_put_object_from_local_file():
Key=file_name
)
assert put_response['ETag'] == etag
response = client.delete_object(
Bucket=test_bucket,
Key=file_name
)
if os.path.exists(file_name):
os.remove(file_name)

Expand Down

0 comments on commit 0be14bc

Please sign in to comment.