From 968b363295b3d643eafd2f8b7c997ebdce1a1a0d Mon Sep 17 00:00:00 2001 From: Jon Skeet Date: Thu, 9 Feb 2017 09:21:17 +0000 Subject: [PATCH] Additional support for storage classes - String constants to avoid clients having to hard-code names - Remove restriction on CopyObject against copying an object to itself (it works and can be useful) - Add the ability to specify extra metadata when copying an object, e.g. to change storage class - A test demonstrating various aspects of storage classes --- .../CopyObjectTest.cs | 32 ++++++-- .../StorageClassTest.cs | 76 +++++++++++++++++++ .../CopyObjectOptions.cs | 8 ++ .../Google.Cloud.Storage.V1/StorageClasses.cs | 59 ++++++++++++++ .../StorageClient.CopyObject.cs | 6 +- .../StorageClientImpl.CopyObject.cs | 7 +- 6 files changed, 171 insertions(+), 17 deletions(-) create mode 100644 apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/StorageClassTest.cs create mode 100644 apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClasses.cs diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/CopyObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/CopyObjectTest.cs index 827321579131..2c107d97cc61 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/CopyObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/CopyObjectTest.cs @@ -15,6 +15,7 @@ using System; using System.Linq; using Xunit; +using Object = Google.Apis.Storage.v1.Data.Object; namespace Google.Cloud.Storage.V1.IntegrationTests { @@ -30,14 +31,6 @@ public CopyObjectTest(StorageFixture fixture) _fixture = fixture; } - [Fact] - public void DestinationEqualsSourceIsProhibited() - { - var bucket = _fixture.ReadBucket; - var name = _fixture.SmallObject; - Assert.Throws(() => _fixture.Client.CopyObject(bucket, name, bucket, name)); - } - [Fact] public void CopySpecificGeneration() { @@ -70,5 +63,28 @@ public void CopyLatestGeneration() ValidateData(_fixture.SingleVersionBucket, destName, _fixture.LargeContent); } + + [Fact] + public void NoMetadataOverride() + { + var destName = GenerateName(); + _fixture.Client.CopyObject( + _fixture.ReadBucket, _fixture.SmallThenLargeObject, + _fixture.SingleVersionBucket, destName); + Object fetched = _fixture.Client.GetObject(_fixture.SingleVersionBucket, destName); + Assert.Equal("text/plain", fetched.ContentType); + } + + [Fact] + public void MetadataOverride() + { + var destName = GenerateName(); + _fixture.Client.CopyObject( + _fixture.ReadBucket, _fixture.SmallThenLargeObject, + _fixture.SingleVersionBucket, destName, + new CopyObjectOptions { ExtraMetadata = new Object { ContentType = "text/html" } }); + Object fetched = _fixture.Client.GetObject(_fixture.SingleVersionBucket, destName); + Assert.Equal("text/html", fetched.ContentType); + } } } diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/StorageClassTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/StorageClassTest.cs new file mode 100644 index 000000000000..944e7843ea30 --- /dev/null +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/StorageClassTest.cs @@ -0,0 +1,76 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Apis.Storage.v1.Data; +using Xunit; +using Object = Google.Apis.Storage.v1.Data.Object; + +namespace Google.Cloud.Storage.V1.IntegrationTests +{ + [Collection(nameof(StorageFixture))] + public class StorageClassTest + { + private readonly StorageFixture _fixture; + + public StorageClassTest(StorageFixture fixture) + { + _fixture = fixture; + } + + // Multi-step test to check: + // - Create a bucket with a storage class of regional + // - That storage class is used when creating a new object + // - If we rewrite the object to a different storage class, that change sticks + // - If we update the bucket's default storage class, creating a new object uses that new class + [Fact] + public void CreateBucketAndObjects() + { + // Note: this test may fail if the project location prevents multi-regional storage. + string initialBucketStorageClass = StorageClasses.MultiRegional; + string updatedObjectStorageClass = StorageClasses.Coldline; + string updatedBucketStorageClass = StorageClasses.Nearline; + + string bucketName = _fixture.BucketPrefix + "storage-classes"; + string objectName = TestHelpers.GenerateName(); + var client = _fixture.Client; + + client.CreateBucket(_fixture.ProjectId, new Bucket { Name = bucketName, StorageClass = initialBucketStorageClass }); + _fixture.RegisterBucketToDelete(bucketName); + + var bucket = client.GetBucket(bucketName); + Assert.Equal(initialBucketStorageClass, bucket.StorageClass); + + var obj = client.UploadObject(bucketName, objectName, "application/octet-stream", TestHelpers.GenerateData(100)); + Assert.Equal(initialBucketStorageClass, obj.StorageClass); + + // Change storage class via a rewrite + var options = new CopyObjectOptions + { + ExtraMetadata = new Object { StorageClass = updatedObjectStorageClass } + }; + client.CopyObject(bucketName, objectName, bucketName, objectName, options); + + // Fetch separately rather than trusting the return value of CopyObject... + obj = client.GetObject(bucketName, objectName); + Assert.Equal(updatedObjectStorageClass, obj.StorageClass); + + bucket.StorageClass = updatedBucketStorageClass; + client.UpdateBucket(bucket); + string objectName2 = TestHelpers.GenerateName(); + + var obj2 = client.UploadObject(bucketName, objectName2, "application/octet-stream", TestHelpers.GenerateData(100)); + Assert.Equal(updatedBucketStorageClass, obj2.StorageClass); + } + } +} diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CopyObjectOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CopyObjectOptions.cs index c2d588e1d7be..302369733a84 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CopyObjectOptions.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CopyObjectOptions.cs @@ -16,6 +16,7 @@ using System; using static Google.Apis.Storage.v1.ObjectsResource; using static Google.Apis.Storage.v1.ObjectsResource.RewriteRequest; +using Object = Google.Apis.Storage.v1.Data.Object; namespace Google.Cloud.Storage.V1 { @@ -90,6 +91,13 @@ public sealed class CopyObjectOptions /// public long? IfSourceMetagenerationNotMatch { get; set; } + /// + /// Additional object metadata for the new object. This can be used to specify the storage + /// class of the new object, the content type etc. If this property is not set, the existing + /// object metadata will be used unchanged. + /// + public Object ExtraMetadata { get; set; } + internal void ModifyRequest(RewriteRequest request) { // Note the use of ArgumentException here, as this will basically be the result of invalid diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClasses.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClasses.cs new file mode 100644 index 000000000000..f442b0acaff7 --- /dev/null +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClasses.cs @@ -0,0 +1,59 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Apis.Storage.v1.Data; + +namespace Google.Cloud.Storage.V1 +{ + /// + /// String constants for the names of the storage classes, as used in + /// and . See https://cloud.google.com/storage/docs/storage-classes for details. + /// + public static class StorageClasses + { + /// + /// Name for the Multi-Regional storage class. + /// + public const string MultiRegional = "MULTI_REGIONAL"; + + /// + /// Name for the Regional storage class. + /// + public const string Regional = "REGIONAL"; + + /// + /// Name for the Nearline storage class. + /// + public const string Nearline = "NEARLINE"; + + /// + /// Name for the Coldline storage class. + /// + public const string Coldline = "COLDLINE"; + + /// + /// Name of the Standard storage class, which is equivalent to + /// Multi-Regional or Regional depending on the location. + /// + public const string Standard = "STANDARD"; + + /// + /// Name of the Durable Reduced Availability (DRA) storage class. + /// Use of this storage class is not recommended; Regional storage has + /// lower pricing for some operations but otherwise the same pricing structure, + /// and better performance/availability. + /// + public const string DurableReducedAvailability = "DURABLE_REDUCED_AVAILABILITY"; + } +} diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClient.CopyObject.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClient.CopyObject.cs index 887cbfade006..5fed2a7b4904 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClient.CopyObject.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClient.CopyObject.cs @@ -29,9 +29,8 @@ public abstract partial class StorageClient /// The name of the object to copy within the bucket. Must not be null. /// The name of the bucket to copy the object to. Must not be null. /// The name of the object within the destination bucket. Must not be null. - /// Additional options for the fetch operation. May be null, in which case appropriate + /// Additional options for the copy operation. May be null, in which case appropriate /// defaults will be used. - /// The arguments attempt to copy an object to itself. /// The representation of the new storage object resulting from the copy. public virtual Object CopyObject( string sourceBucket, @@ -51,9 +50,8 @@ public virtual Object CopyObject( /// The name of the object to copy within the bucket. Must not be null. /// The name of the bucket to copy the object to. Must not be null. /// The name of the object within the destination bucket. Must not be null. - /// Additional options for the fetch operation. May be null, in which case appropriate + /// Additional options for the copy operation. May be null, in which case appropriate /// defaults will be used. - /// The arguments attempt to copy an object to itself. /// The token to monitor for cancellation requests. /// A task representing the asynchronous operation, with a result returning the /// representation of the new storage object resulting from the copy. diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CopyObject.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CopyObject.cs index c586b97115ba..890338d953d7 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CopyObject.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CopyObject.cs @@ -71,11 +71,8 @@ private ObjectsResource.RewriteRequest CreateCopyObjectRequest( GaxPreconditions.CheckNotNull(sourceObjectName, nameof(sourceObjectName)); GaxPreconditions.CheckNotNull(destinationBucket, nameof(destinationBucket)); GaxPreconditions.CheckNotNull(destinationObjectName, nameof(destinationObjectName)); - if (destinationBucket == sourceBucket && destinationObjectName == sourceObjectName) - { - throw new ArgumentException("Cannot copy an object to itself. Specify either a different destination bucket or a different destination object name"); - } - var request = Service.Objects.Rewrite(new Object(), sourceBucket, sourceObjectName, destinationBucket, destinationObjectName); + Object obj = options?.ExtraMetadata ?? new Object(); + var request = Service.Objects.Rewrite(obj, sourceBucket, sourceObjectName, destinationBucket, destinationObjectName); options?.ModifyRequest(request); return request; }