diff --git a/.gitignore b/.gitignore index 57cfd9945..211557ab3 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ functional/.classpath functional/.project functional/.settings/ functional/bin/ -functional/d1/ +functional/.d*/ minio !minio/ minio.exe diff --git a/api/src/main/java/io/minio/GenericResponse.java b/api/src/main/java/io/minio/GenericResponse.java index eb39172d3..5d0308235 100644 --- a/api/src/main/java/io/minio/GenericResponse.java +++ b/api/src/main/java/io/minio/GenericResponse.java @@ -21,12 +21,30 @@ /** Generic response class of any APIs. */ public abstract class GenericResponse { private Headers headers; + private String bucket; + private String region; + private String object; - protected void setHeaders(Headers headers) { + public GenericResponse(Headers headers, String bucket, String region, String object) { this.headers = headers; + this.bucket = bucket; + this.region = region; + this.object = object; } public Headers headers() { return headers; } + + public String bucket() { + return bucket; + } + + public String region() { + return region; + } + + public String object() { + return object; + } } diff --git a/api/src/main/java/io/minio/ListObjectsArgs.java b/api/src/main/java/io/minio/ListObjectsArgs.java index 82ac76c80..951d1ad2e 100644 --- a/api/src/main/java/io/minio/ListObjectsArgs.java +++ b/api/src/main/java/io/minio/ListObjectsArgs.java @@ -18,50 +18,77 @@ /** Argument class of @see #listObjects(ListObjectsArgs args). */ public class ListObjectsArgs extends BucketArgs { - private String continuationToken; - private String delimiter; - private boolean fetchOwner; - private Integer maxKeys; - private String prefix; - private String startAfter; - private boolean includeUserMetadata; + private String delimiter = ""; + private boolean useUrlEncodingType = true; + private String keyMarker; // 'marker' for ListObjectsV1 and 'startAfter' for ListObjectsV2. + private int maxKeys = 1000; + private String prefix = ""; + private String continuationToken; // only for ListObjectsV2. + private boolean fetchOwner; // only for ListObjectsV2. + private String versionIdMarker; // only for GetObjectVersions. + private boolean includeUserMetadata; // MinIO extension applicable to ListObjectsV2. private boolean recursive; - private boolean useVersion1; + private boolean useApiVersion1; + private boolean includeVersions; - public String continuationToken() { - return continuationToken; + public String delimiter() { + if (recursive) { + return ""; + } + + return (delimiter.isEmpty() ? "/" : delimiter); } - public boolean includeUserMetadata() { - return includeUserMetadata; + public boolean useUrlEncodingType() { + return useUrlEncodingType; + } + + public String keyMarker() { + return keyMarker; + } + + public String marker() { + return keyMarker; } public String startAfter() { - return startAfter; + return keyMarker; + } + + public int maxKeys() { + return maxKeys; } public String prefix() { return prefix; } - public Integer maxKeys() { - return maxKeys; + public String continuationToken() { + return continuationToken; } public boolean fetchOwner() { return fetchOwner; } - public String delimiter() { - return delimiter; + public String versionIdMarker() { + return versionIdMarker; + } + + public boolean includeUserMetadata() { + return includeUserMetadata; } public boolean recursive() { return recursive; } - public boolean useVersion1() { - return useVersion1; + public boolean useApiVersion1() { + return useApiVersion1; + } + + public boolean includeVersions() { + return includeVersions; } public static Builder builder() { @@ -70,28 +97,67 @@ public static Builder builder() { /** Argument builder of @see MinioClient#listObjects(ListObjectArgs args). */ public static final class Builder extends BucketArgs.Builder { - public Builder continuationToken(String continuationToken) { - operations.add(args -> args.continuationToken = continuationToken); + @Override + protected void validate(ListObjectsArgs args) { + super.validate(args); + + if (args.useApiVersion1() || args.includeVersions()) { + if (args.continuationToken() != null || args.fetchOwner() || args.includeUserMetadata()) { + throw new IllegalArgumentException( + "continuation token/fetch owner/include metadata are supported only" + + " for list objects version 2"); + } + } + + if (args.versionIdMarker != null && args.useApiVersion1()) { + throw new IllegalArgumentException( + "version ID marker is not supported for list objects version 1"); + } + } + + public Builder delimiter(String delimiter) { + operations.add(args -> args.delimiter = (delimiter == null ? "" : delimiter)); return this; } - public Builder includeUserMetadata(boolean includeUserMetadata) { - operations.add(args -> args.includeUserMetadata = includeUserMetadata); + public Builder useUrlEncodingType(boolean flag) { + operations.add(args -> args.useUrlEncodingType = flag); + return this; + } + + public Builder keyMarker(String keyMarker) { + validateNullOrNotEmptyString(keyMarker, "key marker"); + operations.add(args -> args.keyMarker = keyMarker); + return this; + } + + public Builder marker(String marker) { + operations.add(args -> args.keyMarker = marker); return this; } public Builder startAfter(String startAfter) { - operations.add(args -> args.startAfter = startAfter); + operations.add(args -> args.keyMarker = startAfter); + return this; + } + + public Builder maxKeys(int maxKeys) { + if (maxKeys < 1 || maxKeys > 1000) { + throw new IllegalArgumentException("max keys must be between 1 and 1000"); + } + + operations.add(args -> args.maxKeys = maxKeys); return this; } public Builder prefix(String prefix) { - operations.add(args -> args.prefix = prefix); + operations.add(args -> args.prefix = (prefix == null ? "" : prefix)); return this; } - public Builder maxKeys(Integer maxKeys) { - operations.add(args -> args.maxKeys = maxKeys); + public Builder continuationToken(String continuationToken) { + validateNullOrNotEmptyString(continuationToken, "continuation token"); + operations.add(args -> args.continuationToken = continuationToken); return this; } @@ -100,8 +166,14 @@ public Builder fetchOwner(boolean fetchOwner) { return this; } - public Builder delimiter(String delimiter) { - operations.add(args -> args.delimiter = delimiter); + public Builder versionIdMarker(String versionIdMarker) { + validateNullOrNotEmptyString(versionIdMarker, "version ID marker"); + operations.add(args -> args.versionIdMarker = versionIdMarker); + return this; + } + + public Builder includeUserMetadata(boolean includeUserMetadata) { + operations.add(args -> args.includeUserMetadata = includeUserMetadata); return this; } @@ -110,8 +182,13 @@ public Builder recursive(boolean recursive) { return this; } - public Builder useVersion1(boolean useVersion1) { - operations.add(args -> args.useVersion1 = useVersion1); + public Builder useApiVersion1(boolean useApiVersion1) { + operations.add(args -> args.useApiVersion1 = useApiVersion1); + return this; + } + + public Builder includeVersions(boolean includeVersions) { + operations.add(args -> args.includeVersions = includeVersions); return this; } } diff --git a/api/src/main/java/io/minio/MinioClient.java b/api/src/main/java/io/minio/MinioClient.java index 850da4148..9be6207b4 100755 --- a/api/src/main/java/io/minio/MinioClient.java +++ b/api/src/main/java/io/minio/MinioClient.java @@ -48,6 +48,7 @@ import io.minio.messages.CopyPartResult; import io.minio.messages.CreateBucketConfiguration; import io.minio.messages.DeleteError; +import io.minio.messages.DeleteMarker; import io.minio.messages.DeleteObject; import io.minio.messages.DeleteRequest; import io.minio.messages.DeleteResult; @@ -57,11 +58,12 @@ import io.minio.messages.Item; import io.minio.messages.LegalHold; import io.minio.messages.ListAllMyBucketsResult; -import io.minio.messages.ListBucketResult; import io.minio.messages.ListBucketResultV1; import io.minio.messages.ListBucketResultV2; import io.minio.messages.ListMultipartUploadsResult; +import io.minio.messages.ListObjectsResult; import io.minio.messages.ListPartsResult; +import io.minio.messages.ListVersionsResult; import io.minio.messages.LocationConstraint; import io.minio.messages.NotificationConfiguration; import io.minio.messages.NotificationRecords; @@ -3508,7 +3510,7 @@ public Iterable> listObjects( final String prefix, final boolean recursive, final boolean useVersion1) { - return listObjects(bucketName, prefix, recursive, false, false); + return listObjects(bucketName, prefix, recursive, false, useVersion1); } /** @@ -3550,32 +3552,47 @@ public Iterable> listObjects( .prefix(prefix) .recursive(recursive) .includeUserMetadata(includeUserMetadata) + .useApiVersion1(useVersion1) .build()); } /** - * Lists objects information of a bucket. Supports both the versions 1 and 2 of the S3 API. By - * default, the - * version 2 API is used.
- * version 1 + * Lists objects information optionally with versions of a bucket. Supports both the versions 1 + * and 2 of the S3 API. By default, the version 2 API + * is used.
+ * Version 1 * can be used by passing the optional argument {@code useVersion1} as {@code true}. * - *
Example:
-   * {@code
-   *   Iterable> results = minioClient.listObjects(
+   * 
Example:{@code
+   * // Lists objects information.
+   * Iterable> results = minioClient.listObjects(
+   *     ListObjectsArgs.builder().bucket("my-bucketname").build());
+   *
+   * // Lists objects information recursively.
+   * Iterable> results = minioClient.listObjects(
+   *     ListObjectsArgs.builder().bucket("my-bucketname").recursive(true).build());
+   *
+   * // Lists maximum 100 objects information those names starts with 'E' and after
+   * // 'ExampleGuide.pdf'.
+   * Iterable> results = minioClient.listObjects(
    *     ListObjectsArgs.builder()
-   *       .bucket("my-bucketname")
-   *       .includeUserMetadata(true)
-   *       .startAfter("start-after-entry")
-   *       .prefix("my-obj")
-   *       .maxKeys(100)
-   *       .fetchOwner(true)
-   *   );
-   *   for (Result result : results) {
-   *     Item item = result.get();
-   *     System.out.println(
-   *       item.lastModified() + ", " + item.size() + ", " + item.objectName());
-   *   }
+   *         .bucket("my-bucketname")
+   *         .startAfter("ExampleGuide.pdf")
+   *         .prefix("E")
+   *         .maxKeys(100)
+   *         .build());
+   *
+   * // Lists maximum 100 objects information with version those names starts with 'E' and after
+   * // 'ExampleGuide.pdf'.
+   * Iterable> results = minioClient.listObjects(
+   *     ListObjectsArgs.builder()
+   *         .bucket("my-bucketname")
+   *         .startAfter("ExampleGuide.pdf")
+   *         .prefix("E")
+   *         .maxKeys(100)
+   *         .includeVersions(true)
+   *         .build());
    * }
* * @param args Instance of {@link ListObjectsArgs} built using the builder @@ -3583,36 +3600,26 @@ public Iterable> listObjects( * @throws XmlParserException upon parsing response xml */ public Iterable> listObjects(ListObjectsArgs args) { - if (args.useVersion1()) { - if (args.includeUserMetadata()) { - throw new IllegalArgumentException( - "include user metadata flag is not supported in version 1"); - } + if (args.includeVersions() || args.versionIdMarker() != null) { + return listObjectVersions(args); + } + + if (args.useApiVersion1()) { return listObjectsV1(args); - } else { - return listObjectsV2(args); } + + return listObjectsV2(args); } private abstract class ObjectIterator implements Iterator> { protected Result error; - protected Iterator itemIterator; + protected Iterator itemIterator; + protected Iterator deleteMarkerIterator; protected Iterator prefixIterator; protected boolean completed = false; - protected ListBucketResult listBucketResult; + protected ListObjectsResult listObjectsResult; protected String lastObjectName; - protected String getDelimiter(ListObjectsArgs args) { - if (args.recursive()) { - return null; - } - String delimiter = args.delimiter(); - if (delimiter == null) { - return "/"; - } - return delimiter; - } - protected abstract void populateResult() throws ErrorResponseException, IllegalArgumentException, InsufficientDataException, InternalException, InvalidBucketNameException, InvalidKeyException, @@ -3634,14 +3641,16 @@ protected synchronized void populate() { | ServerException | XmlParserException e) { this.error = new Result<>(e); - } finally { - if (this.listBucketResult != null) { - this.itemIterator = this.listBucketResult.contents().iterator(); - this.prefixIterator = this.listBucketResult.commonPrefixes().iterator(); - } else { - this.itemIterator = new LinkedList().iterator(); - this.prefixIterator = new LinkedList().iterator(); - } + } + + if (this.listObjectsResult != null) { + this.itemIterator = this.listObjectsResult.contents().iterator(); + this.deleteMarkerIterator = this.listObjectsResult.deleteMarkers().iterator(); + this.prefixIterator = this.listObjectsResult.commonPrefixes().iterator(); + } else { + this.itemIterator = new LinkedList().iterator(); + this.deleteMarkerIterator = new LinkedList().iterator(); + this.prefixIterator = new LinkedList().iterator(); } } @@ -3651,14 +3660,18 @@ public boolean hasNext() { return false; } - if (this.error == null && this.itemIterator == null && this.prefixIterator == null) { + if (this.error == null + && this.itemIterator == null + && this.deleteMarkerIterator == null + && this.prefixIterator == null) { populate(); } if (this.error == null && !this.itemIterator.hasNext() + && !this.deleteMarkerIterator.hasNext() && !this.prefixIterator.hasNext() - && this.listBucketResult.isTruncated()) { + && this.listObjectsResult.isTruncated()) { populate(); } @@ -3670,6 +3683,10 @@ public boolean hasNext() { return true; } + if (this.deleteMarkerIterator.hasNext()) { + return true; + } + if (this.prefixIterator.hasNext()) { return true; } @@ -3684,14 +3701,18 @@ public Result next() { throw new NoSuchElementException(); } - if (this.error == null && this.itemIterator == null && this.prefixIterator == null) { + if (this.error == null + && this.itemIterator == null + && this.deleteMarkerIterator == null + && this.prefixIterator == null) { populate(); } if (this.error == null && !this.itemIterator.hasNext() + && !this.deleteMarkerIterator.hasNext() && !this.prefixIterator.hasNext() - && this.listBucketResult.isTruncated()) { + && this.listObjectsResult.isTruncated()) { populate(); } @@ -3705,6 +3726,11 @@ public Result next() { this.lastObjectName = item.objectName(); return new Result<>(item); } + + if (this.deleteMarkerIterator.hasNext()) { + return new Result<>(this.deleteMarkerIterator.next()); + } + if (this.prefixIterator.hasNext()) { return new Result<>(this.prefixIterator.next().toItem()); } @@ -3724,31 +3750,33 @@ private Iterable> listObjectsV2(ListObjectsArgs args) { @Override public Iterator> iterator() { return new ObjectIterator() { + private ListBucketResultV2 result = null; + + @Override protected void populateResult() throws ErrorResponseException, IllegalArgumentException, InsufficientDataException, InternalException, InvalidBucketNameException, InvalidKeyException, InvalidResponseException, IOException, NoSuchAlgorithmException, ServerException, XmlParserException { - String delimiter = getDelimiter(args); - String continuationToken = null; - if (this.listBucketResult != null) { - continuationToken = listBucketResult.nextContinuationToken(); - } - - this.listBucketResult = null; + this.listObjectsResult = null; this.itemIterator = null; this.prefixIterator = null; - this.listBucketResult = - invokeListObjectsV2( - ListObjectsArgs.builder() - .bucket(args.bucket()) - .continuationToken(continuationToken) - .delimiter(delimiter) - .fetchOwner(args.fetchOwner()) - .prefix(args.prefix()) - .includeUserMetadata(args.includeUserMetadata()) - .build()); + result = + listObjectsV2( + args.bucket(), + args.region(), + args.delimiter(), + args.useUrlEncodingType(), + args.startAfter(), + args.maxKeys(), + args.prefix(), + (result == null) ? args.continuationToken() : result.nextContinuationToken(), + args.fetchOwner(), + args.includeUserMetadata(), + args.extraHeaders(), + args.extraQueryParams()); + this.listObjectsResult = result; } }; } @@ -3760,34 +3788,71 @@ private Iterable> listObjectsV1(ListObjectsArgs args) { @Override public Iterator> iterator() { return new ObjectIterator() { + private ListBucketResultV1 result = null; + @Override protected void populateResult() throws ErrorResponseException, IllegalArgumentException, InsufficientDataException, InternalException, InvalidBucketNameException, InvalidKeyException, InvalidResponseException, IOException, NoSuchAlgorithmException, ServerException, XmlParserException { - String delimiter = getDelimiter(args); - String continuationToken = null; - if (this.listBucketResult != null) { - if (delimiter != null) { - continuationToken = listBucketResult.nextContinuationToken(); - } else { - continuationToken = this.lastObjectName; - } + this.listObjectsResult = null; + this.itemIterator = null; + this.prefixIterator = null; + + String nextMarker = (result == null) ? args.marker() : result.nextMarker(); + if (nextMarker == null) { + nextMarker = this.lastObjectName; } - this.listBucketResult = null; + result = + listObjectsV1( + args.bucket(), + args.region(), + args.delimiter(), + args.useUrlEncodingType(), + nextMarker, + args.maxKeys(), + args.prefix(), + args.extraHeaders(), + args.extraQueryParams()); + this.listObjectsResult = result; + } + }; + } + }; + } + + private Iterable> listObjectVersions(ListObjectsArgs args) { + return new Iterable>() { + @Override + public Iterator> iterator() { + return new ObjectIterator() { + private ListVersionsResult result = null; + + @Override + protected void populateResult() + throws ErrorResponseException, IllegalArgumentException, InsufficientDataException, + InternalException, InvalidBucketNameException, InvalidKeyException, + InvalidResponseException, IOException, NoSuchAlgorithmException, ServerException, + XmlParserException { + this.listObjectsResult = null; this.itemIterator = null; this.prefixIterator = null; - this.listBucketResult = - invokeListObjectsV1( - ListObjectsArgs.builder() - .bucket(args.bucket()) - .delimiter(args.delimiter()) - .startAfter(continuationToken) - .prefix(args.prefix()) - .build()); + result = + listObjectVersions( + args.bucket(), + args.region(), + args.delimiter(), + args.useUrlEncodingType(), + (result == null) ? args.keyMarker() : result.nextKeyMarker(), + args.maxKeys(), + args.prefix(), + (result == null) ? args.versionIdMarker() : result.nextVersionIdMarker(), + args.extraHeaders(), + args.extraQueryParams()); + this.listObjectsResult = result; } }; } @@ -7463,8 +7528,6 @@ protected ObjectWriteResponse completeMultipartUpload( queryParams, new CompleteMultipartUpload(parts), 0)) { - String etag = null; - String bodyContent = new String(response.body().bytes(), StandardCharsets.UTF_8); bodyContent = bodyContent.trim(); if (!bodyContent.isEmpty()) { @@ -7480,7 +7543,13 @@ protected ObjectWriteResponse completeMultipartUpload( try { CompleteMultipartUploadOutput result = Xml.unmarshal(CompleteMultipartUploadOutput.class, bodyContent); - etag = result.etag(); + return new ObjectWriteResponse( + response.headers(), + result.bucket(), + result.location(), + result.object(), + result.etag(), + response.header("x-amz-version-id")); } catch (XmlParserException e) { // As this CompleteMultipartUpload REST call succeeded, just log it. Logger.getLogger(MinioClient.class.getName()) @@ -7490,7 +7559,13 @@ protected ObjectWriteResponse completeMultipartUpload( } } - return new ObjectWriteResponse(response.headers(), etag, response.header("x-amz-version-id")); + return new ObjectWriteResponse( + response.headers(), + bucketName, + region, + objectName, + null, + response.header("x-amz-version-id")); } } @@ -7642,72 +7717,144 @@ protected DeleteResult deleteObjects( } } - private Map getCommonListObjectsQueryParams(ListObjectsArgs args) { - Map queryParamMap = new HashMap<>(); - - if (args.delimiter() != null) { - queryParamMap.put("delimiter", args.delimiter()); - } else { - queryParamMap.put("delimiter", ""); - } - - if (args.maxKeys() != null) { - queryParamMap.put("max-keys", args.maxKeys().toString()); - } - - if (args.prefix() != null) { - queryParamMap.put("prefix", args.prefix()); - } else { - queryParamMap.put("prefix", ""); + private Multimap getCommonListObjectsQueryParams( + String delimiter, boolean useUrlEncodingType, int maxKeys, String prefix) { + Multimap queryParams = HashMultimap.create(); + queryParams.put("delimiter", (delimiter == null) ? "" : delimiter); + if (useUrlEncodingType) { + queryParams.put("encoding-type", "url"); } - - return queryParamMap; + queryParams.put("max-keys", Integer.toString(maxKeys > 0 ? maxKeys : 1000)); + queryParams.put("prefix", (prefix == null) ? "" : prefix); + return queryParams; } - private ListBucketResultV2 invokeListObjectsV2(ListObjectsArgs args) + protected ListBucketResultV2 listObjectsV2( + String bucketName, + String region, + String delimiter, + boolean useUrlEncodingType, + String startAfter, + int maxKeys, + String prefix, + String continuationToken, + boolean fetchOwner, + boolean includeUserMetadata, + Multimap extraHeaders, + Multimap extraQueryParams) throws InvalidKeyException, InvalidBucketNameException, IllegalArgumentException, NoSuchAlgorithmException, InsufficientDataException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException, IOException { - Map queryParamMap = getCommonListObjectsQueryParams(args); - queryParamMap.put("list-type", "2"); - - if (args.continuationToken() != null) { - queryParamMap.put("continuation-token", args.continuationToken()); + Multimap queryParams = HashMultimap.create(); + if (extraQueryParams != null) { + queryParams.putAll(extraQueryParams); } - - if (args.fetchOwner()) { - queryParamMap.put("fetch-owner", "true"); + queryParams.putAll( + getCommonListObjectsQueryParams(delimiter, useUrlEncodingType, maxKeys, prefix)); + queryParams.put("list-type", "2"); + if (continuationToken != null) { + queryParams.put("continuation-token", continuationToken); } - - if (args.startAfter() != null) { - queryParamMap.put("start-after", args.startAfter()); + if (fetchOwner) { + queryParams.put("fetch-owner", "true"); } - - if (args.includeUserMetadata()) { - queryParamMap.put("metadata", "true"); + if (startAfter != null) { + queryParams.put("start-after", startAfter); + } + if (includeUserMetadata) { + queryParams.put("metadata", "true"); } - Response response = executeGet(args.bucket(), null, null, queryParamMap); - - try (ResponseBody body = response.body()) { - return Xml.unmarshal(ListBucketResultV2.class, body.charStream()); + try (Response response = + execute( + Method.GET, + bucketName, + null, + (region == null) ? getRegion(bucketName) : region, + extraHeaders, + queryParams, + null, + 0)) { + return Xml.unmarshal(ListBucketResultV2.class, response.body().charStream()); } } - private ListBucketResultV1 invokeListObjectsV1(ListObjectsArgs args) + protected ListBucketResultV1 listObjectsV1( + String bucketName, + String region, + String delimiter, + boolean useUrlEncodingType, + String marker, + int maxKeys, + String prefix, + Multimap extraHeaders, + Multimap extraQueryParams) throws InvalidBucketNameException, IllegalArgumentException, NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException { - Map queryParamMap = getCommonListObjectsQueryParams(args); + Multimap queryParams = HashMultimap.create(); + if (extraQueryParams != null) { + queryParams.putAll(extraQueryParams); + } + queryParams.putAll( + getCommonListObjectsQueryParams(delimiter, useUrlEncodingType, maxKeys, prefix)); + if (marker != null) { + queryParams.put("marker", marker); + } - if (args.startAfter() != null) { - queryParamMap.put("marker", args.startAfter()); + try (Response response = + execute( + Method.GET, + bucketName, + null, + (region == null) ? getRegion(bucketName) : region, + extraHeaders, + queryParams, + null, + 0)) { + return Xml.unmarshal(ListBucketResultV1.class, response.body().charStream()); } + } - Response response = executeGet(args.bucket(), null, null, queryParamMap); + protected ListVersionsResult listObjectVersions( + String bucketName, + String region, + String delimiter, + boolean useUrlEncodingType, + String keyMarker, + int maxKeys, + String prefix, + String versionIdMarker, + Multimap extraHeaders, + Multimap extraQueryParams) + throws InvalidBucketNameException, IllegalArgumentException, NoSuchAlgorithmException, + InsufficientDataException, IOException, InvalidKeyException, ServerException, + XmlParserException, ErrorResponseException, InternalException, InvalidResponseException { + Multimap queryParams = HashMultimap.create(); + if (extraQueryParams != null) { + queryParams.putAll(extraQueryParams); + } + queryParams.putAll( + getCommonListObjectsQueryParams(delimiter, useUrlEncodingType, maxKeys, prefix)); + if (keyMarker != null) { + queryParams.put("key-marker", keyMarker); + } + if (versionIdMarker != null) { + queryParams.put("version-id-marker", versionIdMarker); + } + queryParams.put("versions", ""); - try (ResponseBody body = response.body()) { - return Xml.unmarshal(ListBucketResultV1.class, body.charStream()); + try (Response response = + execute( + Method.GET, + bucketName, + null, + (region == null) ? getRegion(bucketName) : region, + extraHeaders, + queryParams, + null, + 0)) { + return Xml.unmarshal(ListVersionsResult.class, response.body().charStream()); } } @@ -7805,6 +7952,9 @@ protected ObjectWriteResponse putObject( length)) { return new ObjectWriteResponse( response.headers(), + bucketName, + region, + objectName, response.header("ETag").replaceAll("\"", ""), response.header("x-amz-version-id")); } diff --git a/api/src/main/java/io/minio/ObjectWriteResponse.java b/api/src/main/java/io/minio/ObjectWriteResponse.java index 85491abbd..1083a1252 100644 --- a/api/src/main/java/io/minio/ObjectWriteResponse.java +++ b/api/src/main/java/io/minio/ObjectWriteResponse.java @@ -23,8 +23,9 @@ public class ObjectWriteResponse extends GenericResponse { private String etag; private String versionId; - public ObjectWriteResponse(Headers headers, String etag, String versionId) { - setHeaders(headers); + public ObjectWriteResponse( + Headers headers, String bucket, String region, String object, String etag, String versionId) { + super(headers, bucket, region, object); this.etag = etag; this.versionId = versionId; } diff --git a/api/src/main/java/io/minio/messages/Contents.java b/api/src/main/java/io/minio/messages/Contents.java new file mode 100644 index 000000000..873e800e7 --- /dev/null +++ b/api/src/main/java/io/minio/messages/Contents.java @@ -0,0 +1,34 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.messages; + +import org.simpleframework.xml.Root; + +/** + * Helper class to denote Object information in {@link ListBucketResultV1} and {@link + * ListBucketResultV2} + */ +@Root(name = "Contents", strict = false) +public class Contents extends Item { + public Contents() { + super(); + } + + public Contents(String prefix) { + super(prefix); + } +} diff --git a/api/src/main/java/io/minio/messages/DeleteMarker.java b/api/src/main/java/io/minio/messages/DeleteMarker.java new file mode 100644 index 000000000..0e95b74ed --- /dev/null +++ b/api/src/main/java/io/minio/messages/DeleteMarker.java @@ -0,0 +1,31 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.messages; + +import org.simpleframework.xml.Root; + +/** Helper class to denote delete marker information in {@link ListVersionsResult}. */ +@Root(name = "DeleteMarker", strict = false) +public class DeleteMarker extends Item { + public DeleteMarker() { + super(); + } + + public DeleteMarker(String prefix) { + super(prefix); + } +} diff --git a/api/src/main/java/io/minio/messages/Item.java b/api/src/main/java/io/minio/messages/Item.java index edff21c92..79b3c0052 100644 --- a/api/src/main/java/io/minio/messages/Item.java +++ b/api/src/main/java/io/minio/messages/Item.java @@ -19,31 +19,35 @@ import java.time.ZonedDateTime; import java.util.Map; import org.simpleframework.xml.Element; -import org.simpleframework.xml.Root; /** - * Helper class to denote Object information in {@link ListBucketResult} and {@link - * ListBucketResultV1}. + * Helper class to denote Object information in {@link ListBucketResultV1}, {@link + * ListBucketResultV2} and {@link ListVersionsResult}. */ -@Root(name = "Contents", strict = false) -public class Item { +public abstract class Item { + @Element(name = "ETag", required = false) + private String etag; // except DeleteMarker + @Element(name = "Key") private String objectName; @Element(name = "LastModified") private ResponseDate lastModified; - @Element(name = "ETag") - private String etag; + @Element(name = "Owner", required = false) + private Owner owner; + + @Element(name = "Size", required = false) + private long size; // except DeleteMarker - @Element(name = "Size") - private long size; + @Element(name = "StorageClass", required = false) + private String storageClass; // except DeleteMarker - @Element(name = "StorageClass") - private String storageClass; + @Element(name = "IsLatest", required = false) + private boolean isLatest; // except ListObjects V1 - @Element(name = "Owner", required = false) - private Owner owner; + @Element(name = "VersionId", required = false) + private String versionId; // except ListObjects V1 @Element(name = "UserMetadata", required = false) private Metadata userMetadata; @@ -90,15 +94,26 @@ public Owner owner() { /** Returns user metadata. This is MinIO specific extension to ListObjectsV2. */ public Map userMetadata() { - if (userMetadata == null) { - return null; - } + return (userMetadata == null) ? null : userMetadata.get(); + } + + /** Returns whether this version ID is latest. */ + public boolean isLatest() { + return isLatest; + } - return userMetadata.get(); + /** Returns version ID. */ + public String versionId() { + return versionId; } - /** Returns whether the object is a directory or not. */ + /** Returns whether this item is a directory or not. */ public boolean isDir() { return isDir; } + + /** Returns whether this item is a delete marker or not. */ + public boolean isDeleteMarker() { + return (etag == null && size == 0 && storageClass == null && versionId != null); + } } diff --git a/api/src/main/java/io/minio/messages/ListBucketResultV1.java b/api/src/main/java/io/minio/messages/ListBucketResultV1.java index d1f8ec702..e62ca634f 100644 --- a/api/src/main/java/io/minio/messages/ListBucketResultV1.java +++ b/api/src/main/java/io/minio/messages/ListBucketResultV1.java @@ -16,7 +16,9 @@ package io.minio.messages; +import java.util.List; import org.simpleframework.xml.Element; +import org.simpleframework.xml.ElementList; import org.simpleframework.xml.Namespace; import org.simpleframework.xml.Root; @@ -26,22 +28,26 @@ */ @Root(name = "ListBucketResult", strict = false) @Namespace(reference = "http://s3.amazonaws.com/doc/2006-03-01/") -public class ListBucketResultV1 extends ListBucketResult { +public class ListBucketResultV1 extends ListObjectsResult { @Element(name = "Marker", required = false) private String marker; @Element(name = "NextMarker", required = false) private String nextMarker; - /** Returns continuation token. */ - @Override - public String continuationToken() { + @ElementList(name = "Contents", inline = true, required = false) + private List contents; + + public String marker() { return marker; } - /** Returns next continuation token. */ - @Override - public String nextContinuationToken() { + public String nextMarker() { return nextMarker; } + + @Override + public List contents() { + return emptyIfNull(contents); + } } diff --git a/api/src/main/java/io/minio/messages/ListBucketResultV2.java b/api/src/main/java/io/minio/messages/ListBucketResultV2.java index 34541a430..29630047e 100644 --- a/api/src/main/java/io/minio/messages/ListBucketResultV2.java +++ b/api/src/main/java/io/minio/messages/ListBucketResultV2.java @@ -16,7 +16,9 @@ package io.minio.messages; +import java.util.List; import org.simpleframework.xml.Element; +import org.simpleframework.xml.ElementList; import org.simpleframework.xml.Namespace; import org.simpleframework.xml.Root; @@ -27,22 +29,45 @@ */ @Root(name = "ListBucketResult", strict = false) @Namespace(reference = "http://s3.amazonaws.com/doc/2006-03-01/") -public class ListBucketResultV2 extends ListBucketResult { +public class ListBucketResultV2 extends ListObjectsResult { + @Element(name = "KeyCount", required = false) + private int keyCount; + + @Element(name = "StartAfter", required = false) + private String startAfter; + @Element(name = "ContinuationToken", required = false) private String continuationToken; @Element(name = "NextContinuationToken", required = false) private String nextContinuationToken; + @ElementList(name = "Contents", inline = true, required = false) + private List contents; + + /** Returns key count. */ + public int keyCount() { + return keyCount; + } + + /** Returns start after. */ + public String startAfter() { + return startAfter; + } + /** Returns continuation token. */ - @Override public String continuationToken() { return continuationToken; } /** Returns next continuation token. */ - @Override public String nextContinuationToken() { return nextContinuationToken; } + + /** Returns List of Items. */ + @Override + public List contents() { + return emptyIfNull(contents); + } } diff --git a/api/src/main/java/io/minio/messages/ListBucketResult.java b/api/src/main/java/io/minio/messages/ListObjectsResult.java similarity index 62% rename from api/src/main/java/io/minio/messages/ListBucketResult.java rename to api/src/main/java/io/minio/messages/ListObjectsResult.java index c0527d933..a36638e90 100644 --- a/api/src/main/java/io/minio/messages/ListBucketResult.java +++ b/api/src/main/java/io/minio/messages/ListObjectsResult.java @@ -1,5 +1,5 @@ /* - * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc. + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,67 +16,54 @@ package io.minio.messages; +import com.google.common.base.MoreObjects; import java.util.Collections; import java.util.LinkedList; import java.util.List; import org.simpleframework.xml.Element; import org.simpleframework.xml.ElementList; -public abstract class ListBucketResult { +public abstract class ListObjectsResult { @Element(name = "Name") private String name; + @Element(name = "EncodingType", required = false) + private String encodingType; + @Element(name = "Prefix", required = false) private String prefix; - @Element(name = "StartAfter", required = false) - private String startAfter; - - @Element(name = "KeyCount", required = false) - private int keyCount; - - @Element(name = "MaxKeys") - private int maxKeys; - @Element(name = "Delimiter", required = false) private String delimiter; @Element(name = "IsTruncated", required = false) private boolean isTruncated; - @ElementList(name = "Contents", inline = true, required = false) - private List contents; + @Element(name = "MaxKeys", required = false) + private int maxKeys; @ElementList(name = "CommonPrefixes", inline = true, required = false) private List commonPrefixes; - public ListBucketResult() {} + private static final List deleteMarkers = + Collections.unmodifiableList(new LinkedList<>()); + + public ListObjectsResult() {} /** Returns bucket name. */ public String name() { return name; } + public boolean useUrlEncodingType() { + return (encodingType != null && encodingType.equals("url")); + } + /** Returns prefix. */ public String prefix() { return prefix; } - /** Returns start after. */ - public String startAfter() { - return startAfter; - } - - /** Returns key count. */ - public int keyCount() { - return keyCount; - } - - /** Returns max keys. */ - public int maxKeys() { - return maxKeys; - } - /** Returns delimiter. */ public String delimiter() { return delimiter; @@ -87,26 +74,24 @@ public boolean isTruncated() { return isTruncated; } - /** Returns List of Items. */ - public List contents() { - if (contents == null) { - return Collections.unmodifiableList(new LinkedList<>()); - } - - return Collections.unmodifiableList(contents); + /** Returns max keys. */ + public int maxKeys() { + return maxKeys; } /** Returns List of Prefix. */ public List commonPrefixes() { - if (commonPrefixes == null) { - return Collections.unmodifiableList(new LinkedList<>()); - } + return Collections.unmodifiableList( + (commonPrefixes == null) ? new LinkedList<>() : commonPrefixes); + } - return Collections.unmodifiableList(commonPrefixes); + public List deleteMarkers() { + return deleteMarkers; } - /** Returns continuation token. */ - public abstract String continuationToken(); + protected List emptyIfNull(List lst) { + return Collections.unmodifiableList(MoreObjects.firstNonNull(lst, new LinkedList())); + } - public abstract String nextContinuationToken(); + public abstract List contents(); } diff --git a/api/src/main/java/io/minio/messages/ListVersionsResult.java b/api/src/main/java/io/minio/messages/ListVersionsResult.java new file mode 100644 index 000000000..da65f0a31 --- /dev/null +++ b/api/src/main/java/io/minio/messages/ListVersionsResult.java @@ -0,0 +1,76 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.messages; + +import java.util.List; +import org.simpleframework.xml.Element; +import org.simpleframework.xml.ElementList; +import org.simpleframework.xml.Namespace; +import org.simpleframework.xml.Root; + +/** + * Object representation of response XML of ListObjectVersions + * API. + */ +@Root(name = "ListVersionsResult", strict = false) +@Namespace(reference = "http://s3.amazonaws.com/doc/2006-03-01/") +public class ListVersionsResult extends ListObjectsResult { + @Element(name = "KeyMarker", required = false) + private String keyMarker; + + @Element(name = "NextKeyMarker", required = false) + private String nextKeyMarker; + + @Element(name = "VersionIdMarker", required = false) + private String versionIdMarker; + + @Element(name = "NextVersionIdMarker", required = false) + private String nextVersionIdMarker; + + @ElementList(name = "Version", inline = true, required = false) + private List contents; + + @ElementList(name = "DeleteMarker", inline = true, required = false) + private List deleteMarkers; + + public String keyMarker() { + return keyMarker; + } + + public String nextKeyMarker() { + return nextKeyMarker; + } + + public String versionIdMarker() { + return versionIdMarker; + } + + public String nextVersionIdMarker() { + return nextVersionIdMarker; + } + + @Override + public List contents() { + return emptyIfNull(contents); + } + + @Override + public List deleteMarkers() { + return emptyIfNull(deleteMarkers); + } +} diff --git a/api/src/main/java/io/minio/messages/Owner.java b/api/src/main/java/io/minio/messages/Owner.java index a10d123b7..28ffa998b 100644 --- a/api/src/main/java/io/minio/messages/Owner.java +++ b/api/src/main/java/io/minio/messages/Owner.java @@ -21,8 +21,8 @@ /** * Helper class to denote owner information for {@link ListAllMyBucketsResult}, {@link - * ListBucketResult}, {@link ListBucketResultV1}, {@link ListMultipartUploadsResult} and {@link - * ListPartsResult}. + * ListBucketResultV1}, {@link ListBucketResultV2}, {@link ListVersionsResult}, {@link + * ListMultipartUploadsResult} and {@link ListPartsResult}. */ @Root(name = "Owner", strict = false) public class Owner { diff --git a/api/src/main/java/io/minio/messages/Prefix.java b/api/src/main/java/io/minio/messages/Prefix.java index bc2f4343f..4a27a7bc4 100644 --- a/api/src/main/java/io/minio/messages/Prefix.java +++ b/api/src/main/java/io/minio/messages/Prefix.java @@ -20,8 +20,8 @@ import org.simpleframework.xml.Root; /** - * Helper class to denote Prefix information in {@link ListBucketResult} and {@link - * ListBucketResultV1}. + * Helper class to denote Prefix information in {@link ListBucketResultV1}, {@link + * ListBucketResultV2} and {@link ListVersionsResult}. */ @Root(name = "CommonPrefixes", strict = false) public class Prefix { @@ -31,6 +31,6 @@ public class Prefix { public Prefix() {} public Item toItem() { - return new Item(prefix); + return new Contents(prefix); } } diff --git a/api/src/main/java/io/minio/messages/Version.java b/api/src/main/java/io/minio/messages/Version.java new file mode 100644 index 000000000..65eb57d97 --- /dev/null +++ b/api/src/main/java/io/minio/messages/Version.java @@ -0,0 +1,31 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio.messages; + +import org.simpleframework.xml.Root; + +/** Helper class to denote object and it's version information in {@link ListVersionsResult}. */ +@Root(name = "Version", strict = false) +public class Version extends Item { + public Version() { + super(); + } + + public Version(String prefix) { + super(prefix); + } +} diff --git a/docs/API.md b/docs/API.md index 698e9ef04..a61d0ffae 100644 --- a/docs/API.md +++ b/docs/API.md @@ -727,19 +727,33 @@ __Parameters__ __Example__ ```java +// Lists objects information. Iterable> results = minioClient.listObjects( - ListObjectsArgs.builder() - .bucket("my-bucketname") - .includeUserMetadata(true) - .startAfter("start-after-entry") - .prefix("my-obj") - .maxKeys(100) - .fetchOwner(true) -); -for (Result result : results) { - Item item = result.get(); - System.out.println(item.lastModified() + ", " + item.size() + ", " + item.objectName()); -} + ListObjectsArgs.builder().bucket("my-bucketname").build()); + +// Lists objects information recursively. +Iterable> results = minioClient.listObjects( + ListObjectsArgs.builder().bucket("my-bucketname").recursive(true).build()); + +// Lists maximum 100 objects information those names starts with 'E' and after 'ExampleGuide.pdf'. +Iterable> results = minioClient.listObjects( + ListObjectsArgs.builder() + .bucket("my-bucketname") + .startAfter("ExampleGuide.pdf") + .prefix("E") + .maxKeys(100) + .build()); + +// Lists maximum 100 objects information with version those names starts with 'E' and after +// 'ExampleGuide.pdf'. +Iterable> results = minioClient.listObjects( + ListObjectsArgs.builder() + .bucket("my-bucketname") + .startAfter("ExampleGuide.pdf") + .prefix("E") + .maxKeys(100) + .includeVersions(true) + .build()); ``` ### makeBucket(MakeBucketArgs args) diff --git a/examples/ListObjects.java b/examples/ListObjects.java index 208237a82..7ee6443fc 100644 --- a/examples/ListObjects.java +++ b/examples/ListObjects.java @@ -14,7 +14,6 @@ * limitations under the License. */ -import io.minio.BucketExistsArgs; import io.minio.ListObjectsArgs; import io.minio.MinioClient; import io.minio.Result; @@ -40,19 +39,72 @@ public static void main(String[] args) // MinioClient minioClient = new MinioClient("https://s3.amazonaws.com", "YOUR-ACCESSKEYID", // "YOUR-SECRETACCESSKEY"); - // Check whether 'my-bucketname' exist or not. - boolean found = - minioClient.bucketExists(BucketExistsArgs.builder().bucket("my-bucketname").build()); - if (found) { - // List objects from 'my-bucketname' - Iterable> myObjects = + { + // Lists objects information. + Iterable> results = minioClient.listObjects(ListObjectsArgs.builder().bucket("my-bucketname").build()); - for (Result result : myObjects) { + + for (Result result : results) { + Item item = result.get(); + System.out.println(item.lastModified() + "\t" + item.size() + "\t" + item.objectName()); + } + } + + { + // Lists objects information recursively. + Iterable> results = + minioClient.listObjects( + ListObjectsArgs.builder().bucket("my-bucketname").recursive(true).build()); + + for (Result result : results) { + Item item = result.get(); + System.out.println(item.lastModified() + "\t" + item.size() + "\t" + item.objectName()); + } + } + + { + // Lists maximum 100 objects information those names starts with 'E' and after + // 'ExampleGuide.pdf'. + Iterable> results = + minioClient.listObjects( + ListObjectsArgs.builder() + .bucket("my-bucketname") + .startAfter("ExampleGuide.pdf") + .prefix("E") + .maxKeys(100) + .build()); + + for (Result result : results) { + Item item = result.get(); + System.out.println(item.lastModified() + "\t" + item.size() + "\t" + item.objectName()); + } + } + + { + // Lists maximum 100 objects information with version those names starts with 'E' and after + // 'ExampleGuide.pdf'. + Iterable> results = + minioClient.listObjects( + ListObjectsArgs.builder() + .bucket("my-bucketname") + .startAfter("ExampleGuide.pdf") + .prefix("E") + .maxKeys(100) + .includeVersions(true) + .build()); + + for (Result result : results) { Item item = result.get(); - System.out.println(item.lastModified() + ", " + item.size() + ", " + item.objectName()); + System.out.println( + item.lastModified() + + "\t" + + item.size() + + "\t" + + item.objectName() + + " [" + + item.versionId() + + "]"); } - } else { - System.out.println("my-bucketname does not exist"); } } catch (MinioException e) { System.out.println("Error occurred: " + e); diff --git a/functional/FunctionalTest.java b/functional/FunctionalTest.java index 4693e48d6..485917105 100644 --- a/functional/FunctionalTest.java +++ b/functional/FunctionalTest.java @@ -56,6 +56,7 @@ import io.minio.MakeBucketArgs; import io.minio.MinioClient; import io.minio.ObjectStat; +import io.minio.ObjectWriteResponse; import io.minio.PostPolicy; import io.minio.PutObjectArgs; import io.minio.RemoveBucketArgs; @@ -1356,25 +1357,37 @@ public static void downloadObject_test() throws Exception { .build()); } - public static String[] createObjects(String bucketName, int count) throws Exception { - String[] objectNames = new String[count]; + public static List createObjects(String bucketName, int count, int versions) + throws Exception { + List results = new LinkedList<>(); for (int i = 0; i < count; i++) { - objectNames[i] = getRandomName(); - client.putObject( - PutObjectArgs.builder().bucket(bucketName).object(objectNames[i]).stream( - new ContentInputStream(1), 1, -1) - .build()); + String objectName = getRandomName(); + results.add( + client.putObject( + PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( + new ContentInputStream(1), 1, -1) + .build())); + if (versions > 1) { + for (int j = 0; j < versions - 1; j++) { + results.add( + client.putObject( + PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( + new ContentInputStream(1), 1, -1) + .build())); + } + } } - return objectNames; + return results; } - public static void removeObjects(String bucketName, String[] objectNames) throws Exception { + public static void removeObjects(String bucketName, List results) + throws Exception { List objects = - Arrays.stream(objectNames) + results.stream() .map( - name -> { - return new DeleteObject(name); + result -> { + return new DeleteObject(result.object(), result.versionId()); }) .collect(Collectors.toList()); for (Result r : @@ -1385,18 +1398,19 @@ public static void removeObjects(String bucketName, String[] objectNames) throws } public static void testListObjects( - String methodName, String tag, ListObjectsArgs args, int objCount) throws Exception { - if (!mintEnv) { - System.out.println("Test: " + tag + " " + methodName); - } - + String testTags, ListObjectsArgs args, int objCount, int versions) throws Exception { + String methodName = "listObjects()"; long startTime = System.currentTimeMillis(); String bucketName = args.bucket(); - String[] objectNames = null; + List results = null; try { client.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); try { - objectNames = createObjects(bucketName, objCount); + if (versions > 0) { + client.enableVersioning(EnableVersioningArgs.builder().bucket(bucketName).build()); + } + + results = createObjects(bucketName, objCount, versions); int i = 0; for (Result r : client.listObjects(args)) { @@ -1404,74 +1418,69 @@ public static void testListObjects( i++; } + if (versions > 0) { + objCount *= versions; + } + if (i != objCount) { throw new Exception("object count; expected=" + objCount + ", got=" + i); } - mintSuccessLog(methodName, tag, startTime); + mintSuccessLog(methodName, testTags, startTime); } finally { - if (objectNames != null) { - removeObjects(bucketName, objectNames); + if (results != null) { + removeObjects(bucketName, results); } client.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); } } catch (Exception e) { - handleException(methodName, tag, startTime, e); + handleException(methodName, testTags, startTime, e); } } - /** Test: [bucket] listObjects(ListObjectsArgs args). */ - public static void listObjects_test1() throws Exception { - testListObjects( - "listObjects(ListObjectsArgs args)", - "[bucket]", - ListObjectsArgs.builder().bucket(getRandomName()).build(), - 3); - } + public static void listObjects_test() throws Exception { + if (!mintEnv) { + System.out.println("Test: listObjects()"); + } + + testListObjects("[bucket]", ListObjectsArgs.builder().bucket(getRandomName()).build(), 3, 0); - /** Test: [bucket, prefix] listObjects(ListObjectsArgs args). */ - public static void listObjects_test2() throws Exception { testListObjects( - "listObjects(ListObjectsArgs args)", "[bucket, prefix]", ListObjectsArgs.builder().bucket(getRandomName()).prefix("minio").build(), - 3); - } + 3, + 0); - /** Test: [bucket, prefix, recursive] listObjects(ListObjectsArgs args). */ - public static void listObjects_test3() throws Exception { testListObjects( - "listObjects(ListObjectsArgs args)", "[bucket, prefix, recursive]", ListObjectsArgs.builder().bucket(getRandomName()).prefix("minio").recursive(true).build(), - 3); - } + 3, + 0); - /** Test: [empty bucket] listObjects(ListObjectsArgs args). */ - public static void listObjects_test4() throws Exception { testListObjects( - "listObjects(ListObjectsArgs args)", - "[empty bucket]", - ListObjectsArgs.builder().bucket(getRandomName()).build(), - 0); - } + "[bucket, versions]", + ListObjectsArgs.builder().bucket(getRandomName()).includeVersions(true).build(), + 3, + 2); + + if (isQuickTest) { + return; + } + + testListObjects( + "[empty bucket]", ListObjectsArgs.builder().bucket(getRandomName()).build(), 0, 0); - /** Test: [bucket, prefix, recursive, 1050 objects] listObjects(ListObjectsArgs args). */ - public static void listObjects_test5() throws Exception { testListObjects( - "listObjects(ListObjectsArgs args)", "[bucket, prefix, recursive, 1050 objects]", ListObjectsArgs.builder().bucket(getRandomName()).prefix("minio").recursive(true).build(), - 1050); - } + 1050, + 0); - /** Test: [bucket, version1] listObjects(ListObjectsArgs args). */ - public static void listObjects_test6() throws Exception { testListObjects( - "listObjects(ListObjectsArgs args)", - "[bucket, version1]", - ListObjectsArgs.builder().bucket(getRandomName()).useVersion1(true).build(), - 3); + "[bucket, apiVersion1]", + ListObjectsArgs.builder().bucket(getRandomName()).useApiVersion1(true).build(), + 3, + 0); } /** Test: removeObject(String bucketName, String objectName). */ @@ -1511,18 +1520,17 @@ public static void removeObjects_test1() throws Exception { long startTime = System.currentTimeMillis(); try { String bucketName = getRandomName(); - String[] objectNames = null; client.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build()); + List results = null; try { - String[] createdNames = createObjects(bucketName, 3); - objectNames = new String[createdNames.length + 1]; - System.arraycopy(createdNames, 0, objectNames, 0, createdNames.length); - objectNames[3] = "nonexistent-object"; - removeObjects(bucketName, objectNames); + results = createObjects(bucketName, 3, 0); + results.add( + new ObjectWriteResponse(null, bucketName, null, "nonexistent-object", null, null)); + removeObjects(bucketName, results); mintSuccessLog(methodName, null, startTime); } finally { - if (objectNames != null) { - removeObjects(bucketName, objectNames); + if (results != null) { + removeObjects(bucketName, results); } client.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); } @@ -2808,23 +2816,24 @@ public static void composeObject_test6() throws Exception { } } - /** Test: enableObjectLegalHold(EnableObjectLegalHoldArgs args) */ public static void enableObjectLegalHold_test() throws Exception { - String methodName = "enableObjectLegalHold(EnableObjectLegalHoldArgs args)"; + String methodName = "enableObjectLegalHold()"; if (!mintEnv) { System.out.println("Test: " + methodName); } long startTime = System.currentTimeMillis(); String bucketName = getRandomName(); String objectName = getRandomName(); + ObjectWriteResponse objectInfo = null; try { client.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(true).build()); try { - client.putObject( - PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( - new ContentInputStream(1 * KB), 1 * KB, -1) - .build()); + objectInfo = + client.putObject( + PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( + new ContentInputStream(1 * KB), 1 * KB, -1) + .build()); client.enableObjectLegalHold( EnableObjectLegalHoldArgs.builder().bucket(bucketName).object(objectName).build()); @@ -2836,8 +2845,14 @@ public static void enableObjectLegalHold_test() throws Exception { DisableObjectLegalHoldArgs.builder().bucket(bucketName).object(objectName).build()); mintSuccessLog(methodName, null, startTime); } finally { - client.removeObject( - RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build()); + if (objectInfo != null) { + client.removeObject( + RemoveObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .versionId(objectInfo.versionId()) + .build()); + } client.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); } } catch (Exception e) { @@ -2845,22 +2860,23 @@ public static void enableObjectLegalHold_test() throws Exception { } } - /** Test: disableObjectLegalHold(DisableObjectLegalHoldArgs args) */ public static void disableObjectLegalHold_test() throws Exception { - String methodName = "disableObjectLegalHold(DisableObjectLegalHoldArgs args)"; + String methodName = "disableObjectLegalHold()"; if (!mintEnv) { System.out.println("Test: " + methodName); } long startTime = System.currentTimeMillis(); String bucketName = getRandomName(); String objectName = getRandomName(); + ObjectWriteResponse objectInfo = null; try { client.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(true).build()); try { - client.putObject( - PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( - new ContentInputStream(1 * KB), 1 * KB, -1) - .build()); + objectInfo = + client.putObject( + PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( + new ContentInputStream(1 * KB), 1 * KB, -1) + .build()); client.enableObjectLegalHold( EnableObjectLegalHoldArgs.builder().bucket(bucketName).object(objectName).build()); client.disableObjectLegalHold( @@ -2870,8 +2886,14 @@ public static void disableObjectLegalHold_test() throws Exception { throw new Exception("[FAILED] isObjectLegalHoldEnabled(): expected: false, got: true"); } } finally { - client.removeObject( - RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build()); + if (objectInfo != null) { + client.removeObject( + RemoveObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .versionId(objectInfo.versionId()) + .build()); + } client.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); } mintSuccessLog(methodName, null, startTime); @@ -2981,9 +3003,8 @@ public static void deleteDefaultRetention_test() throws Exception { } } - /** Test: setObjectRetention(SetObjectRetentionArgs args). */ - public static void setObjectRetention_test1() throws Exception { - String methodName = "setObjectRetention(SetObjectRetentionArgs args)"; + public static void setObjectRetention_test() throws Exception { + String methodName = "setObjectRetention()"; if (!mintEnv) { System.out.println("Test: " + methodName); } @@ -2991,13 +3012,15 @@ public static void setObjectRetention_test1() throws Exception { long startTime = System.currentTimeMillis(); String bucketName = getRandomName(); String objectName = getRandomName(); + ObjectWriteResponse objectInfo = null; try { client.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(true).build()); try { - client.putObject( - PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( - new ContentInputStream(1 * KB), 1 * KB, -1) - .build()); + objectInfo = + client.putObject( + PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( + new ContentInputStream(1 * KB), 1 * KB, -1) + .build()); ZonedDateTime retentionUntil = ZonedDateTime.now(Time.UTC).plusDays(1); Retention expectedConfig = new Retention(RetentionMode.GOVERNANCE, retentionUntil); @@ -3018,8 +3041,14 @@ public static void setObjectRetention_test1() throws Exception { .build()); } finally { - client.removeObject( - RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build()); + if (objectInfo != null) { + client.removeObject( + RemoveObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .versionId(objectInfo.versionId()) + .build()); + } client.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); } mintSuccessLog(methodName, null, startTime); @@ -3028,9 +3057,8 @@ public static void setObjectRetention_test1() throws Exception { } } - /** Test: getObjectRetention(GetObjectRetentionArgs args). */ - public static void getObjectRetention_test1() throws Exception { - String methodName = "getObjectRetention(GetObjectRetentionArgs args)"; + public static void getObjectRetention_test() throws Exception { + String methodName = "getObjectRetention()"; if (!mintEnv) { System.out.println("Test: " + methodName); } @@ -3038,13 +3066,15 @@ public static void getObjectRetention_test1() throws Exception { long startTime = System.currentTimeMillis(); String bucketName = getRandomName(); String objectName = getRandomName(); + ObjectWriteResponse objectInfo = null; try { client.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(true).build()); try { - client.putObject( - PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( - new ContentInputStream(1 * KB), 1 * KB, -1) - .build()); + objectInfo = + client.putObject( + PutObjectArgs.builder().bucket(bucketName).object(objectName).stream( + new ContentInputStream(1 * KB), 1 * KB, -1) + .build()); ZonedDateTime retentionUntil = ZonedDateTime.now(Time.UTC).plusDays(3); Retention expectedConfig = new Retention(RetentionMode.GOVERNANCE, retentionUntil); @@ -3123,8 +3153,14 @@ public static void getObjectRetention_test1() throws Exception { .build()); } finally { - client.removeObject( - RemoveObjectArgs.builder().bucket(bucketName).object(objectName).build()); + if (objectInfo != null) { + client.removeObject( + RemoveObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .versionId(objectInfo.versionId()) + .build()); + } client.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); } mintSuccessLog(methodName, null, startTime); @@ -3900,8 +3936,8 @@ public static void runTests() throws Exception { uploadObject_test(); downloadObject_test(); - setObjectRetention_test1(); - getObjectRetention_test1(); + setObjectRetention_test(); + getObjectRetention_test(); statObject_test1(); statObject_test2(); @@ -3915,12 +3951,7 @@ public static void runTests() throws Exception { getPresignedObjectUrl_test5(); getPresignedObjectUrl_test6(); - listObjects_test1(); - listObjects_test2(); - listObjects_test3(); - listObjects_test4(); - listObjects_test5(); - listObjects_test6(); + listObjects_test(); removeObject_test1(); removeObjects_test1(); @@ -3953,8 +3984,8 @@ public static void runTests() throws Exception { setDefaultRetention_test(); getDefaultRetention_test(); - setObjectRetention_test1(); - getObjectRetention_test1(); + setObjectRetention_test(); + getObjectRetention_test(); selectObjectContent_test1(); @@ -4011,7 +4042,7 @@ public static void runQuickTests() throws Exception { statObject_test1(); getObject_test(); downloadObject_test(); - listObjects_test1(); + listObjects_test(); removeObject_test1(); listIncompleteUploads_test1(); removeIncompleteUploads_test(); @@ -4088,7 +4119,8 @@ public static boolean downloadMinio() throws IOException { public static Process runMinio() throws Exception { File binaryPath = new File(new File(System.getProperty("user.dir")), MINIO_BINARY); - ProcessBuilder pb = new ProcessBuilder(binaryPath.getPath(), "server", "d1"); + ProcessBuilder pb = + new ProcessBuilder(binaryPath.getPath(), "server", ".d1", ".d2", ".d3", ".d4"); Map env = pb.environment(); env.put("MINIO_ACCESS_KEY", "minio");