Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: calculate/add MD5 hash for multiple object delete. #581

Merged
merged 1 commit into from
Jun 2, 2017
Merged

fix: calculate/add MD5 hash for multiple object delete. #581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 54 additions & 52 deletions api/src/main/java/io/minio/Digest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.google.common.io.BaseEncoding;

import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;


/**
Expand All @@ -42,84 +43,85 @@ private Digest() {}
* Returns SHA-256 hash of given string.
*/
public static String sha256Hash(String string) throws NoSuchAlgorithmException {
return sha256Hash(string.getBytes(StandardCharsets.UTF_8));
}


/**
* Returns SHA-256 hash of given byte array.
*/
public static String sha256Hash(byte[] data) throws NoSuchAlgorithmException {
return sha256Hash(data, data.length);
}


/**
* Returns SHA-256 hash string of given byte array and it's length.
*/
public static String sha256Hash(byte[] data, int length) throws NoSuchAlgorithmException {
MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");

messageDigest.update(data, 0, length);

return BaseEncoding.base16().encode(messageDigest.digest()).toLowerCase();
byte[] data = string.getBytes(StandardCharsets.UTF_8);
MessageDigest sha256Digest = MessageDigest.getInstance("SHA-256");
sha256Digest.update((byte[]) data, 0, data.length);
return BaseEncoding.base16().encode(sha256Digest.digest()).toLowerCase();
}


/**
* Returns SHA-256 of given input stream and it's length.
* Returns SHA-256 hash of given data and it's length.
*
* @param inputStream Input stream whose type is either {@link RandomAccessFile} or {@link BufferedInputStream}.
* @param len Length of Input stream.
* @param data must be {@link RandomAccessFile}, {@link BufferedInputStream} or byte array.
* @param len length of data to be read for hash calculation.
*/
public static String sha256Hash(Object inputStream, int len)
throws NoSuchAlgorithmException, IOException, InsufficientDataException {
public static String sha256Hash(Object data, int len)
throws NoSuchAlgorithmException, IOException, InsufficientDataException, InternalException {
MessageDigest sha256Digest = MessageDigest.getInstance("SHA-256");
updateDigests(inputStream, len, sha256Digest, null);
return BaseEncoding.base16().encode(sha256Digest.digest()).toLowerCase();
}

/**
* Returns MD5 hash of given string.
*/
public static String md5Hash(String string) throws NoSuchAlgorithmException {
return md5Hash(string.getBytes(StandardCharsets.UTF_8));
}

if (data instanceof BufferedInputStream || data instanceof RandomAccessFile) {
updateDigests(data, len, sha256Digest, null);
} else if (data instanceof byte[]) {
sha256Digest.update((byte[]) data, 0, len);
} else {
throw new InternalException("Unknown data source to calculate sha256 hash. This should not happen, "
+ "please report this issue at https://github.com/minio/minio-java/issues");
}

/**
* Returns MD5 hash of given byte array.
*/
public static String md5Hash(byte[] data) throws NoSuchAlgorithmException {
return md5Hash(data, data.length);
return BaseEncoding.base16().encode(sha256Digest.digest()).toLowerCase();
}


/**
* Returns MD5 hash of given byte array and it's length.
* Returns SHA-256 and MD5 hashes of given data and it's length.
*
* @param data must be {@link RandomAccessFile}, {@link BufferedInputStream} or byte array.
* @param len length of data to be read for hash calculation.
*/
public static String md5Hash(byte[] data, int length) throws NoSuchAlgorithmException {
MessageDigest messageDigest = MessageDigest.getInstance("MD5");
public static String[] sha256Md5Hashes(Object data, int len)
throws NoSuchAlgorithmException, IOException, InsufficientDataException, InternalException {
MessageDigest sha256Digest = MessageDigest.getInstance("SHA-256");
MessageDigest md5Digest = MessageDigest.getInstance("MD5");

messageDigest.update(data, 0, length);
if (data instanceof BufferedInputStream || data instanceof RandomAccessFile) {
updateDigests(data, len, sha256Digest, md5Digest);
} else if (data instanceof byte[]) {
sha256Digest.update((byte[]) data, 0, len);
md5Digest.update((byte[]) data, 0, len);
} else {
throw new InternalException("Unknown data source to calculate sha256 hash. This should not happen, "
+ "please report this issue at https://github.com/minio/minio-java/issues");
}

return BaseEncoding.base64().encode(messageDigest.digest());
return new String[]{BaseEncoding.base16().encode(sha256Digest.digest()).toLowerCase(),
BaseEncoding.base64().encode(md5Digest.digest())};
}


/**
* Returns MD5 hash of given input stream and it's length.
* Returns MD5 hash of given data and it's length.
*
* @param inputStream Input stream whose type is either {@link RandomAccessFile} or {@link BufferedInputStream}.
* @param len Length of Input stream.
* @param data must be {@link RandomAccessFile}, {@link BufferedInputStream} or byte array.
* @param len length of data to be read for hash calculation.
*/
public static String md5Hash(Object inputStream, int len)
throws NoSuchAlgorithmException, IOException, InsufficientDataException {
public static String md5Hash(Object data, int len)
throws NoSuchAlgorithmException, IOException, InsufficientDataException, InternalException {
MessageDigest md5Digest = MessageDigest.getInstance("MD5");
updateDigests(inputStream, len, null, md5Digest);

if (data instanceof BufferedInputStream || data instanceof RandomAccessFile) {
updateDigests(data, len, null, md5Digest);
} else if (data instanceof byte[]) {
md5Digest.update((byte[]) data, 0, len);
} else {
throw new InternalException("Unknown data source to calculate sha256 hash. This should not happen, "
+ "please report this issue at https://github.com/minio/minio-java/issues");
}

return BaseEncoding.base64().encode(md5Digest.digest());
}


Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unexpected line?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no. you can see double newlines between methods

/**
* Updated MessageDigest with bytes read from file and stream.
*/
Expand Down
48 changes: 20 additions & 28 deletions api/src/main/java/io/minio/MinioClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,8 @@ private Request createRequest(Method method, String bucketName, String objectNam
String region, Map<String,String> headerMap,
Map<String,String> queryParamMap, final String contentType,
final Object body, final int length)
throws InvalidBucketNameException, NoSuchAlgorithmException, InsufficientDataException, IOException {
throws InvalidBucketNameException, NoSuchAlgorithmException, InsufficientDataException, IOException,
InternalException {
if (bucketName == null && objectName != null) {
throw new InvalidBucketNameException(NULL_STRING, "null bucket name for object '" + objectName + "'");
}
Expand Down Expand Up @@ -896,40 +897,31 @@ public void writeTo(BufferedSink sink) throws IOException {
// Fix issue #415: No need to compute sha256 if endpoint scheme is HTTPS.
if (url.isHttps()) {
sha256Hash = "UNSIGNED-PAYLOAD";
if (body instanceof BufferedInputStream) {
md5Hash = Digest.md5Hash((BufferedInputStream) body, length);
} else if (body instanceof RandomAccessFile) {
md5Hash = Digest.md5Hash((RandomAccessFile) body, length);
} else if (body instanceof byte[]) {
byte[] data = (byte[]) body;
md5Hash = Digest.md5Hash(data, length);
if (body != null) {
md5Hash = Digest.md5Hash(body, length);
}
} else {
// Fix issue #567: Compute SHA256 hash only.
if (body == null) {
sha256Hash = Digest.sha256Hash(new byte[0]);
} else if (body instanceof BufferedInputStream) {
sha256Hash = Digest.sha256Hash((BufferedInputStream) body, length);
} else if (body instanceof RandomAccessFile) {
sha256Hash = Digest.sha256Hash((RandomAccessFile) body, length);
} else if (body instanceof byte[]) {
sha256Hash = Digest.sha256Hash((byte[]) body, length);
Object data = body;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you might want to consider ternary operator for conciseness
Object data = (body == null) ? new byte[0] : body;
int len = (body == null) ? 0 : length;

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For this case, data and len are reset when data == null in one if seems better to me for readability sake.

There is a chance of misread two (body == null) statements.

int len = length;
if (data == null) {
data = new byte[0];
len = 0;
}

if (method == Method.POST && queryParamMap != null && queryParamMap.containsKey("delete")) {
// Fix issue #579: Treat 'Delete Multiple Objects' specially which requires MD5 hash.
String[] hashes = Digest.sha256Md5Hashes(data, len);
sha256Hash = hashes[0];
md5Hash = hashes[1];
} else {
sha256Hash = Digest.sha256Hash(body.toString());
// Fix issue #567: Compute SHA256 hash only.
sha256Hash = Digest.sha256Hash(data, len);
}
}
} else {
// Fix issue #567: Compute MD5 hash only of anonymous access.
if (body != null) {
if (body instanceof BufferedInputStream) {
md5Hash = Digest.md5Hash((BufferedInputStream) body, length);
} else if (body instanceof RandomAccessFile) {
md5Hash = Digest.md5Hash((RandomAccessFile) body, length);
} else if (body instanceof byte[]) {
md5Hash = Digest.md5Hash((byte[]) body, length);
} else {
md5Hash = Digest.md5Hash(body.toString());
}
md5Hash = Digest.md5Hash(body, length);
}
}

Expand Down Expand Up @@ -2081,7 +2073,7 @@ public String presignedPutObject(String bucketName, String objectName, Integer e
}

String region = getRegion(bucketName);
Request request = createRequest(Method.PUT, bucketName, objectName, region, null, null, null, "", 0);
Request request = createRequest(Method.PUT, bucketName, objectName, region, null, null, null, new byte[0], 0);
HttpUrl url = Signer.presignV4(request, region, accessKey, secretKey, expires);
return url.toString();
}
Expand Down