Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

S3 #48

Merged
merged 3 commits into from
May 4, 2018
Merged

S3 #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f1848c6
Modify lifecycle
dt3310321 Apr 25, 2018
c856672
fix some bugs and modify ut
dt3310321 May 4, 2018
7db4cc6
modify UA
dt3310321 May 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion qcloud_cos/cos_auth.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,13 @@ def __call__(self, r):
path = self._path
uri_params = self._params
headers = filter_headers(r.headers)
uri_params = dict([(k.lower(), v.lower()) for k, v in uri_params.items()])
# reserved keywords in headers urlencode are -_.~, notice that / should be encoded and space should not be encoded to plus sign(+)
headers = dict([(k.lower(), quote(v, '-_.~')) for k, v in headers.items()]) # headers中的key转换为小写，value进行encode
format_str = "{method}\n{host}\n{params}\n{headers}\n".format(
method=r.method.lower(),
host=path,
params=urllib.urlencode(sorted(uri_params.items())),
params=urllib.urlencode(sorted(uri_params.items())).replace('+', '%20'),
headers='&'.join(map(lambda (x, y): "%s=%s" % (x, y), sorted(headers.items())))
)
logger.debug("format str: " + format_str)
Expand Down
46 changes: 31 additions & 15 deletions qcloud_cos/cos_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def send_request(self, method, url, timeout=30, **kwargs):
timeout = self._conf._timeout
if self._conf._token is not None:
kwargs['headers']['x-cos-security-token'] = self._conf._token
kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.1'
kwargs['headers']['User-Agent'] = 'cos-python-sdk-v5.1.4.2'
try:
for j in range(self._retry):
if method == 'POST':
Expand Down Expand Up @@ -391,7 +391,7 @@ def delete_objects(self, Bucket, Delete={}, **kwargs):
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
headers=headers)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Deleted', 'Error'])
format_dict(data, ['Deleted', 'Error'])
return data

def head_object(self, Bucket, Key, **kwargs):
Expand Down Expand Up @@ -723,9 +723,9 @@ def list_parts(self, Bucket, Key, UploadId, EncodingType='', MaxParts=1000, Part
headers=headers,
params=params)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Part'])
format_dict(data, ['Part'])
if decodeflag:
data = decode_result(data, ['Key'], [])
decode_result(data, ['Key'], [])
return data

def put_object_acl(self, Bucket, Key, AccessControlPolicy={}, **kwargs):
Expand Down Expand Up @@ -938,9 +938,9 @@ def list_objects(self, Bucket, Prefix="", Delimiter="", Marker="", MaxKeys=1000,
headers=headers,
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
data = xml_to_dict(rt.text)
data = format_dict(data, ['Contents', 'CommonPrefixes'])
format_dict(data, ['Contents', 'CommonPrefixes'])
if decodeflag:
data = decode_result(
decode_result(
data,
[
'Prefix',
Expand Down Expand Up @@ -1006,9 +1006,9 @@ def list_objects_versions(self, Bucket, Prefix="", Delimiter="", KeyMarker="", V
headers=headers,
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))
data = xml_to_dict(rt.text)
data = format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes'])
format_dict(data, ['Version', 'DeleteMarker', 'CommonPrefixes'])
if decodeflag:
data = decode_result(
decode_result(
data,
[
'Prefix',
Expand Down Expand Up @@ -1078,9 +1078,9 @@ def list_multipart_uploads(self, Bucket, Prefix="", Delimiter="", KeyMarker="",
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key))

data = xml_to_dict(rt.text)
data = format_dict(data, ['Upload', 'CommonPrefixes'])
format_dict(data, ['Upload', 'CommonPrefixes'])
if decodeflag:
data = decode_result(
decode_result(
data,
[
'Prefix',
Expand Down Expand Up @@ -1349,7 +1349,17 @@ def put_bucket_lifecycle(self, Bucket, LifecycleConfiguration={}, **kwargs):
LifecycleConfiguration=lifecycle_config
)
"""
lst = ['<Rule>', '<Tag>', '</Tag>', '</Rule>'] # 类型为list的标签
# 类型为list的标签
lst = [
'<Rule>',
'<Tag>',
'<Transition>',
'<NoncurrentVersionTransition>',
'</NoncurrentVersionTransition>',
'</Transition>',
'</Tag>',
'</Rule>'
]
xml_config = format_xml(data=LifecycleConfiguration, root='LifecycleConfiguration', lst=lst)
headers = mapped(kwargs)
headers['Content-MD5'] = get_md5(xml_config)
Expand Down Expand Up @@ -1393,7 +1403,12 @@ def get_bucket_lifecycle(self, Bucket, **kwargs):
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
headers=headers)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Rule'])
format_dict(data, ['Rule'])
if 'Rule' in data.keys():
for rule in data['Rule']:
format_dict(rule, ['Transition', 'NoncurrentVersionTransition'])
if 'Filter' in rule.keys():
format_dict(rule['Filter'], ['Tag'])
return data

def delete_bucket_lifecycle(self, Bucket, **kwargs):
Expand Down Expand Up @@ -1596,7 +1611,7 @@ def get_bucket_replication(self, Bucket, **kwargs):
auth=CosS3Auth(self._conf._secret_id, self._conf._secret_key),
headers=headers)
data = xml_to_dict(rt.text)
data = format_dict(data, ['Rule'])
format_dict(data, ['Rule'])
return data

def delete_bucket_replication(self, Bucket, **kwargs):
Expand Down Expand Up @@ -1814,11 +1829,12 @@ def _check_all_upload_parts(self, bucket, key, uploadid, local_path, parts_num,
UploadId=uploadid,
PartNumberMarker=part_number_marker
)
parts_info.extend(response['Part'])
if 'Part' in response:
parts_info.extend(response['Part'])
if response['IsTruncated'] == 'false':
list_over_status = True
else:
part_number_marker = int(response['NextMarker'])
part_number_marker = int(response['NextPartNumberMarker'])
for part in parts_info:
part_num = int(part['PartNumber'])
# 如果分块数量大于本地计算出的最大数量,校验失败
Expand Down
97 changes: 73 additions & 24 deletions ut/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@

SECRET_ID = os.environ["SECRET_ID"]
SECRET_KEY = os.environ["SECRET_KEY"]
test_bucket = "test01-1252448703"
TRAVIS_FLAG = os.environ["TRAVIS_FLAG"]
test_bucket = 'cos-python-v5-test-' + str(sys.version_info[0]) + '-' + str(sys.version_info[1]) + '-' + '1252448703'
test_object = "test.txt"
special_file_name = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
conf = CosConfig(
Expand All @@ -23,6 +24,19 @@
client = CosS3Client(conf)


def _create_test_bucket(test_bucket):
try:
response = client.create_bucket(
Bucket=test_bucket,
)
except Exception as e:
if e.get_error_code() == 'BucketAlreadyOwnedByYou':
print('BucketAlreadyOwnedByYou')
else:
raise e
return None


def get_raw_md5(data):
m2 = hashlib.md5(data)
etag = '"' + str(m2.hexdigest()) + '"'
Expand All @@ -37,22 +51,24 @@ def gen_file(path, size):


def print_error_msg(e):
print e.get_origin_msg()
print e.get_digest_msg()
print e.get_status_code()
print e.get_error_code()
print e.get_error_msg()
print e.get_resource_location()
print e.get_trace_id()
print e.get_request_id()
print (e.get_origin_msg())
print (e.get_digest_msg())
print (e.get_status_code())
print (e.get_error_code())
print (e.get_error_msg())
print (e.get_resource_location())
print (e.get_trace_id())
print (e.get_request_id())


def setUp():
print "start test..."
print ("start test...")
print ("start create bucket " + test_bucket)
_create_test_bucket(test_bucket)


def tearDown():
print "function teardown"
print ("function teardown")


def test_put_get_delete_object_10MB():
Expand Down Expand Up @@ -146,6 +162,11 @@ def test_put_object_non_exist_bucket():

def test_put_object_acl():
"""设置object acl"""
response = client.put_object(
Bucket=test_bucket,
Key=test_object,
Body='test acl'
)
response = client.put_object_acl(
Bucket=test_bucket,
Key=test_object,
Expand All @@ -160,6 +181,10 @@ def test_get_object_acl():
Key=test_object
)
assert response
response = client.delete_object(
Bucket=test_bucket,
Key=test_object
)


def test_copy_object_diff_bucket():
Expand Down Expand Up @@ -352,7 +377,7 @@ def test_get_bucket_acl_normal():
def test_list_objects():
"""列出bucket下的objects"""
response = client.list_objects(
Bucket=test_bucket,
Bucket='test01-1252448703',
MaxKeys=100,
Prefix='中文',
Delimiter='/'
Expand All @@ -376,7 +401,7 @@ def test_get_presigned_url():
Key='中文.txt'
)
assert url
print url
print (url)


def test_get_bucket_location():
Expand Down Expand Up @@ -430,10 +455,27 @@ def test_put_get_delete_lifecycle():
lifecycle_config = {
'Rule': [
{
'Expiration': {'Date': get_date(2030, 5, 1)},
'ID': '123',
'Filter': {'Prefix': ''},
'Status': 'Enabled',
'Filter': {
# 作用于带标签键 datalevel 和值 backup 的标签的对象
'Tag': [
{
'Key': 'datalevel',
'Value': 'backup'
}
]
},
'Transation': [
{
# 30天后转换为Standard_IA
'Days': 30,
'StorageClass': 'Standard_IA'
}
],
'Expiration': {
# 3650天后过期删除
'Days': 3650
}
}
]
}
Expand Down Expand Up @@ -479,7 +521,7 @@ def test_put_get_delete_replication():
{
'ID': '123',
'Status': 'Enabled',
'Prefix': 'replication',
'Prefix': '中文',
'Destination': {
'Bucket': 'qcs:id/0:cos:cn-south:appid/1252448703:replicationsouth'
}
Expand All @@ -499,7 +541,7 @@ def test_put_get_delete_replication():
Bucket=test_bucket
)
assert response
# delete lifecycle
# delete replication
response = client.delete_bucket_replication(
Bucket=test_bucket
)
Expand Down Expand Up @@ -547,7 +589,10 @@ def test_list_multipart_uploads():
def test_upload_file_multithreading():
"""根据文件大小自动选择分块大小,多线程并发上传提高上传速度"""
file_name = "thread_1GB"
gen_file(file_name, 5) # set 5MB beacuse travis too slow
file_size = 1024
if TRAVIS_FLAG == 'true':
file_size = 5 # set 5MB beacuse travis too slow
gen_file(file_name, file_size)
st = time.time() # 记录开始时间
response = client.upload_file(
Bucket=test_bucket,
Expand All @@ -560,7 +605,7 @@ def test_upload_file_multithreading():
ed = time.time() # 记录结束时间
if os.path.exists(file_name):
os.remove(file_name)
print ed - st
print (ed - st)


def test_copy_file_automatically():
Expand Down Expand Up @@ -608,7 +653,8 @@ def test_use_get_auth():
Key='test.txt',
Params={'acl': '', 'unsed': '123'}
)
response = requests.get('http://test01-1252448703.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123', headers={'Authorization': auth})
url = 'http://' + test_bucket + '.cos.ap-beijing-1.myqcloud.com/test.txt?acl&unsed=123'
response = requests.get(url, headers={'Authorization': auth})
assert response.status_code == 200


Expand Down Expand Up @@ -646,16 +692,15 @@ def test_put_get_bucket_logging():
response = logging_client.get_bucket_logging(
Bucket=logging_bucket
)
print response
print (response)
assert response['LoggingEnabled']['TargetBucket'] == logging_bucket
assert response['LoggingEnabled']['TargetPrefix'] == 'test'


def test_put_object_enable_md5():
"""上传文件,SDK计算content-md5头部"""
file_size = 10
file_name = 'test_object_sdk_caculate_md5.file'
gen_file(file_name, 10)
gen_file(file_name, 1)
with open(file_name, 'rb') as f:
etag = get_raw_md5(f.read())
with open(file_name, 'rb') as fp:
Expand Down Expand Up @@ -686,6 +731,10 @@ def test_put_object_from_local_file():
Key=file_name
)
assert put_response['ETag'] == etag
response = client.delete_object(
Bucket=test_bucket,
Key=file_name
)
if os.path.exists(file_name):
os.remove(file_name)

Expand Down