Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add batch get, update, delete methods to Storage and Blob #233

Merged
merged 6 commits into from
Oct 12, 2015
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

package com.google.gcloud.spi;

import static com.google.common.base.MoreObjects.firstNonNull;

import com.google.api.services.storage.model.Bucket;
import com.google.api.services.storage.model.StorageObject;
import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -106,9 +108,12 @@ class BatchRequest {
public BatchRequest(Iterable<Tuple<StorageObject, Map<Option, ?>>> toDelete,
Iterable<Tuple<StorageObject, Map<Option, ?>>> toUpdate,
Iterable<Tuple<StorageObject, Map<Option, ?>>> toGet) {
this.toDelete = ImmutableList.copyOf(toDelete);
this.toUpdate = ImmutableList.copyOf(toUpdate);
this.toGet = ImmutableList.copyOf(toGet);
this.toDelete = ImmutableList.copyOf(

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

firstNonNull(toDelete, ImmutableList.<Tuple<StorageObject, Map<Option, ?>>>of()));
this.toUpdate = ImmutableList.copyOf(
firstNonNull(toUpdate, ImmutableList.<Tuple<StorageObject, Map<Option, ?>>>of()));
this.toGet = ImmutableList.copyOf(
firstNonNull(toGet, ImmutableList.<Tuple<StorageObject, Map<Option, ?>>>of()));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,17 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.gcloud.storage.Blob.BlobSourceOption.convert;

import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.gcloud.spi.StorageRpc;
import com.google.gcloud.storage.Storage.BlobTargetOption;
import com.google.gcloud.storage.Storage.CopyRequest;
import com.google.gcloud.storage.Storage.SignUrlOption;

import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;

/**
Expand Down Expand Up @@ -256,4 +261,87 @@ public URL signUrl(long expirationTimeInSeconds, SignUrlOption... options) {
public Storage storage() {
return storage;
}

/**
* Gets the requested blobs. If {@code infos.length == 0} an empty list is returned. If

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

* {@code infos.length > 1} a batch request is used to fetch blobs.
*
* @param storage the storage service used to issue the request
* @param infos the blobs to get
* @return an immutable list of {@code Blob} objects. If a blob does not exist or access to it has
* been denied the corresponding item in the list is {@code null}.

This comment was marked as spam.

*/
public static List<Blob> get(final Storage storage, BlobInfo... infos) {
checkNotNull(storage);
checkNotNull(infos);
int length = infos.length;
switch (length) {
case 0:
return Collections.emptyList();
case 1:
return Collections.singletonList(
new Blob(storage, storage.get(infos[0].bucket(), infos[0].name())));

This comment was marked as spam.

default:
return Collections.unmodifiableList(Lists.transform(storage.get(infos),
new Function<BlobInfo, Blob>() {
@Override
public Blob apply(BlobInfo f) {
return f != null ? new Blob(storage, f) : null;
}
}));
}
}

/**
* Updates the requested blobs. If {@code infos.length == 0} an empty list is returned. If
* {@code infos.length > 1} a batch request is used to update blobs.
*
* @param storage the storage service used to issue the request
* @param infos the blobs to update
* @return an immutable list of {@code Blob} objects. If a blob does not exist or access to it has
* been denied the corresponding item in the list is {@code null}.
*/
public static List<Blob> update(final Storage storage, BlobInfo... infos) {
checkNotNull(storage);
checkNotNull(infos);
int length = infos.length;
switch (length) {
case 0:
return Collections.emptyList();

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

case 1:
return Collections.singletonList(new Blob(storage, storage.update(infos[0])));
default:
return Collections.unmodifiableList(Lists.transform(storage.update(infos),
new Function<BlobInfo, Blob>() {
@Override
public Blob apply(BlobInfo f) {
return f != null ? new Blob(storage, f) : null;
}
}));
}
}

/**
* Deletes the requested blobs. If {@code infos.length == 0} an empty list is returned. If
* {@code infos.length > 1} a batch request is used to delete blobs.
*
* @param storage the storage service used to issue the request
* @param infos the blobs to delete
* @return an immutable list of booleans. If a blob has been deleted the corresponding item in the
* list is {@code true}. If deletion failed or access to the resource was denied the item is
* {@code false}.
*/
public static List<Boolean> delete(Storage storage, BlobInfo... infos) {
checkNotNull(storage);
checkNotNull(infos);
int length = infos.length;
switch (length) {
case 0:
return Collections.emptyList();
case 1:
return Collections.singletonList(storage.delete(infos[0].bucket(), infos[0].name()));
default:
return Collections.unmodifiableList(storage.delete(infos));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.gcloud.storage.Storage.BucketTargetOption;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;

Expand Down Expand Up @@ -146,21 +147,26 @@ public Blob get(String blob, BlobSourceOption... options) {
/**
* Returns a list of requested blobs in this bucket. Blobs that do not exist are null.
*
* @param blobNames names of the requested blobs
* @param blobName1 first blob to get
* @param blobName2 second blob to get
* @param blobNames other blobs to get
* @return an immutable list of {@code Blob} objects.
* @throws StorageException upon failure
*/
public List<Blob> getAll(String... blobNames) {
public List<Blob> get(String blobName1, String blobName2, String... blobNames) {
BatchRequest.Builder batch = BatchRequest.builder();
for (String blobName : blobNames) {
batch.get(info.name(), blobName);
batch.get(info.name(), blobName1);
batch.get(info.name(), blobName2);
for (String name : blobNames) {
batch.get(info.name(), name);
}
List<Blob> blobs = new ArrayList<>(blobNames.length);
BatchResponse response = storage.apply(batch.build());
for (BatchResponse.Result<BlobInfo> result : response.gets()) {
BlobInfo blobInfo = result.get();
blobs.add(blobInfo != null ? new Blob(storage, blobInfo) : null);
}
return blobs;
return Collections.unmodifiableList(blobs);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,14 @@ public static Builder builder() {
*/
BlobInfo update(BlobInfo blobInfo, BlobTargetOption... options);

/**
* Update blob information.
*
* @return the updated blob
* @throws StorageException upon failure
*/
BlobInfo update(BlobInfo blobInfo);

/**
* Delete the requested bucket.
*
Expand Down Expand Up @@ -638,4 +646,32 @@ public static Builder builder() {
* @see <a href="https://cloud.google.com/storage/docs/access-control#Signed-URLs">Signed-URLs</a>
*/
URL signUrl(BlobInfo blobInfo, long expirationTimeInSeconds, SignUrlOption... options);

/**
* Gets the requested blobs. A batch request is used to perform this call.
*
* @param blobInfos blobs to get
* @return an immutable list of {@code BlobInfo} objects. If a blob does not exist or access to it

This comment was marked as spam.

* has been denied the corresponding item in the list is {@code null}.
*/
List<BlobInfo> get(BlobInfo... blobInfos);

/**
* Updates the requested blobs. A batch request is used to perform this call.
*
* @param blobInfos blobs to update
* @return an immutable list of {@code BlobInfo} objects. If a blob does not exist or access to it
* has been denied the corresponding item in the list is {@code null}.
*/
List<BlobInfo> update(BlobInfo... blobInfos);

/**
* Deletes the requested blobs. A batch request is used to perform this call.
*
* @param blobInfos blobs to delete
* @return an immutable list of booleans. If a blob has been deleted the corresponding item in the
* list is {@code true}. If deletion failed or access to the resource was denied the item is
* {@code false}.
*/
List<Boolean> delete(BlobInfo... blobInfos);
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
import java.security.Signature;
import java.security.SignatureException;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -348,6 +349,11 @@ public StorageObject call() {
}
}

@Override
public BlobInfo update(BlobInfo blobInfo) {
return update(blobInfo, new BlobTargetOption[0]);
}

@Override
public boolean delete(String bucket, BucketSourceOption... options) {
final com.google.api.services.storage.model.Bucket bucketPb = BucketInfo.of(bucket).toPb();
Expand Down Expand Up @@ -577,6 +583,46 @@ public URL signUrl(BlobInfo blobInfo, long expiration, SignUrlOption... options)
}
}

@Override
public List<BlobInfo> get(BlobInfo... blobInfos) {
BatchRequest.Builder requestBuilder = BatchRequest.builder();
for (BlobInfo blobInfo : blobInfos) {
requestBuilder.get(blobInfo.bucket(), blobInfo.name());
}
BatchResponse response = apply(requestBuilder.build());
return Collections.unmodifiableList(transformResultList(response.gets(), null));
}

@Override
public List<BlobInfo> update(BlobInfo... blobInfos) {
BatchRequest.Builder requestBuilder = BatchRequest.builder();
for (BlobInfo blobInfo : blobInfos) {
requestBuilder.update(blobInfo);
}
BatchResponse response = apply(requestBuilder.build());
return Collections.unmodifiableList(transformResultList(response.updates(), null));
}

@Override
public List<Boolean> delete(BlobInfo... blobInfos) {
BatchRequest.Builder requestBuilder = BatchRequest.builder();
for (BlobInfo blobInfo : blobInfos) {
requestBuilder.delete(blobInfo.bucket(), blobInfo.name());
}
BatchResponse response = apply(requestBuilder.build());
return Collections.unmodifiableList(transformResultList(response.deletes(), Boolean.FALSE));
}

private static <T extends Serializable> List<T> transformResultList(
List<BatchResponse.Result<T>> results, final T errorValue) {
return Lists.transform(results, new Function<BatchResponse.Result<T>, T>() {
@Override
public T apply(BatchResponse.Result<T> f) {
return f.failed() ? errorValue : f.get();
}
});
}

private Map<StorageRpc.Option, ?> optionMap(Long generation, Long metaGeneration,
Iterable<? extends Option> options) {
return optionMap(generation, metaGeneration, options, false);
Expand Down
Loading