[Tables] Add authentication via SAS credential (#15256)

* changing recording to use test-proxy

* updating client and test proxy

* updated Transport Do to send actual request

* whitespace

* working implementation

* updating internal recording

* now I have a blank recording...

* cleaning up

* more clean up

* transport Do is repeating, proxy is not parsing URL correctly

* small changes

* adding a policy to do the same thing

* adding a second test for proxy and transport

* further progress with Jeff, switching to only using a policy right now

* cleaning up file

* cleaning up

* added more debugging, added two required headers that I was missing

* proxy working on a single test, need to convert all tests to test proxy

* cleaning up

* moved over successfully :)

* more helper functions

* working for a subtest functionality

* fixed up a service client test

* converting more service tests

* converted all service client tests

* converted table client tests

* fixed last two client tests

* entity test

* entity tests

* converting access policy tests

* all passing in record mode

* batch tests

* liveonly for batch tests

* adding a sanitizer and more methods for recording

* adding main method to start and stop the proxy automatically

* adding test proxy step to pipeline

* double dash on version arg

* need a better way to get userful error reporting

* explicitly running test-proxy in the background

* adjust the proxy start and install to be compatible with specific process

* dont run test proxy in background

* didnt remove an import

* header to proxy_test.go

* fixes for pipeline

* added loggin on accident

* use nohup on linux machines

* further simplify nohup usage

* there are no exes on windows

* fixing errors, passing in default env variable

* force background for linux task. may need another iteration to place it within the quotes

* fake aad credential

* clean up after test-proxy

* add print for record mode

* another different attempt at starting the proxy

* jobs apparently is an unrecognized command on our linux boxes

* correct dumb typo in run tests

* replace nuget install with docker

* working fake credentials for client delete entity test

* fixing fake credentials portion, passing locally

* trying an insecureskipverify transport

* debugging statements to find the cwd

* removing blank recording

* working implementation of sas

* working sas credential for account signature

* table sas is failing

* leverage proxy tool, not docker

* remove apostrophes

* moving recordings one directory up

* transitioning to using the sas written by storage team

* using storage methods

* copied storage code, sas needs to be appended in a different way

* changing directory for test-proxy to start from

* removing parenthesis

* adding steps for validating ssl

* switching to docker

* adding start server script

* working implementation if we can fix the prepending of 'recording' to the docker request finder

* big refactor, moving stuff into recording file, cleaning up proxy_test.go file

* last fixings

* convert back to docker. windows and linux images now present

* double condition

* moving configuration to a separate file. preparing for eng/common move

* tier 0 of trust. I don't expect this to work, but it's still worth a shot. maybe the devops agents do something differently

* update condition to use variable syntax

* guess I'm not using variables. removing

* call start-server.ps1

* adjust build-test and configure-proxy to run the docker container in context

* wrap in quotes for the container create. it's apparently a bad

* adjust the initialize call

* disable vet temporarily

* looking for cert file in env variable PROXY_CERT

* set proxy_cert environment variable to find that certificate

* changing to crt file

* use crt cert

* want to see errors

* powershell errors

* correcting how the volume binds to the windows container

* small modification for windows container

* finally got the magic sauce

* cleaning up to remove azcore from internal

* removing recordings that use vcr

* issue with the url creation

* persist query params correctly in azcore.JoinPaths

* removing print statements

* return root

* changing location of script

* forgot the stop command

* working table level sas implementation

* adding start/end rk/pk functionality

* removing print statements

* removing more non-tables code

* docstrings and removing storage only code

* making sas tests live only

* Fixing sas table name to be lowercase always, adding test to verify read only

* Fixing sas table name to be lowercase always, adding test to verify read only

* adding cosmos sas test

* fix for params

* apiview fixes

* fixing test

* undoing change

* updates

* error naming

* fixed service client, was adding percallpolicies twice

* simplification

* aligning with main

* converting from query pager to list pager for naming consistency

* updating objects for consistency

* jeffs feedback

* small fixes

* autorest.md changes

* fixing tablename

Co-authored-by: scbedd <[email protected]>

eng/pipelines/templates/steps/build-test.yml

@@ -37,7 +37,7 @@ steps:
       foreach ($td in $testDirs) {
         pushd $td
 
-        $(Build.SourcesDirectory)/eng/scripts/start-server.ps1 start
+        $(Build.SourcesDirectory)/eng/scripts/proxy-server.ps1 start
 
         Write-Host "##[command]Executing go test -run "^Test" -v -coverprofile coverage.txt $td | go-junit-report -set-exit-code > report.xml"
         go test -run "^Test" -v -coverprofile coverage.txt . > temp.txt
@@ -49,7 +49,7 @@ steps:
           rm coverage.txt
         }
 
-        $(Build.SourcesDirectory)/eng/scripts/start-server.ps1 stop
+        $(Build.SourcesDirectory)/eng/scripts/proxy-server.ps1 stop
       }
     displayName: 'Run Tests'
     workingDirectory: '${{parameters.GoWorkspace}}'

sdk/azcore/request.go

@@ -67,23 +67,29 @@ func (ov opValues) get(value interface{}) bool {
 }
 
 // JoinPaths concatenates multiple URL path segments into one path,
-// inserting path separation characters as required.
-func JoinPaths(paths ...string) string {
+// inserting path separation characters as required. JoinPaths will preserve
+// query parameters in the root path
+func JoinPaths(root string, paths ...string) string {
 	if len(paths) == 0 {
-		return ""
-	}
-	path := paths[0]
-	for i := 1; i < len(paths); i++ {
-		if path[len(path)-1] == '/' && paths[i][0] == '/' {
-			// strip off trailing '/' to avoid doubling up
-			path = path[:len(path)-1]
-		} else if path[len(path)-1] != '/' && paths[i][0] != '/' {
-			// add a trailing '/'
-			path = path + "/"
-		}
-		path += paths[i]
+		return root
+	}
+
+	qps := ""
+	if strings.Contains(root, "?") {
+		splitPath := strings.Split(root, "?")
+		root, qps = splitPath[0], splitPath[1]
+	}
+
+	for i := 0; i < len(paths); i++ {
+		root = strings.TrimRight(root, "/")
+		paths[i] = strings.TrimLeft(paths[i], "/")
+		root += "/" + paths[i]
+	}
+
+	if qps != "" {
+		return root + "?" + qps
 	}
-	return path
+	return root
 }
 
 // NewRequest creates a new Request with the specified input.

sdk/azcore/request_test.go

@@ -507,7 +507,7 @@ func TestNewRequestFail(t *testing.T) {
 }
 
 func TestJoinPaths(t *testing.T) {
-	if path := JoinPaths(); path != "" {
+	if path := JoinPaths(""); path != "" {
 		t.Fatalf("unexpected path %s", path)
 	}
 	const expected = "http://test.contoso.com/path/one/path/two/path/three/path/four/"

sdk/tables/autorest.md

@@ -11,12 +11,15 @@ version: "^3.0.0"
 input-file: https://github.com/Azure/azure-rest-api-specs/blob/d744b6bcb95ab4034832ded556dbbe58f4287c5b/specification/cosmos-db/data-plane/Microsoft.Tables/preview/2019-02-02/table.json
 license-header: MICROSOFT_MIT_NO_VERSION
 clear-output-folder: false
-output-folder: aztable
-file-prefix: "zz_generated_"
+output-folder: aztable/internal
+# file-prefix: "zz_generated_"
 tag: package-2019-02
 credential-scope: none
-use: "@autorest/[email protected]"
-openapi-type: data-plane
+use: "@autorest/[email protected]"
+# openapi-type: data-plane
+module-version: 0.1.0
+modelerfour:
+  group-parameters: false
 ```
 
 ### Go multi-api

sdk/tables/aztable/byte_array_response.go

@@ -9,7 +9,7 @@ import (
 	"time"
 )
 
-// ByteArrayResponse converts the MapOfInterfaceResponse.Value from a map[string]interface{} to a []byte
+// ByteArrayResponse is the return type for a GetEntity operation. The entities properties are stored in the Value property
 type ByteArrayResponse struct {
 	// ClientRequestID contains the information returned from the x-ms-client-request-id header response.
 	ClientRequestID *string
@@ -45,6 +45,7 @@ type ByteArrayResponse struct {
 	XMSContinuationNextRowKey *string
 }
 
+// newByteArrayResponse converts a MapofInterfaceResponse from a map[string]interface{} to a []byte.
 func newByteArrayResponse(m MapOfInterfaceResponse) (ByteArrayResponse, error) {
 	marshalledValue, err := json.Marshal(m.Value)
 	if err != nil {
@@ -65,8 +66,8 @@ func newByteArrayResponse(m MapOfInterfaceResponse) (ByteArrayResponse, error) {
 	}, nil
 }
 
-// TableEntityQueryByteResponseResponse is the response envelope for operations that return a TableEntityQueryResponse type.
-type TableEntityQueryByteResponseResponse struct {
+// TableEntityListByteResponseResponse is the response envelope for operations that return a TableEntityQueryResponse type.
+type TableEntityListByteResponseResponse struct {
 	// ClientRequestID contains the information returned from the x-ms-client-request-id header response.
 	ClientRequestID *string
 
@@ -101,12 +102,12 @@ type TableEntityQueryByteResponse struct {
 	Value [][]byte
 }
 
-func castToByteResponse(resp *TableEntityQueryResponseResponse) (TableEntityQueryByteResponseResponse, error) {
+func castToByteResponse(resp *TableEntityQueryResponseResponse) (TableEntityListByteResponseResponse, error) {
 	marshalledValue := make([][]byte, 0)
 	for _, e := range resp.TableEntityQueryResponse.Value {
 		m, err := json.Marshal(e)
 		if err != nil {
-			return TableEntityQueryByteResponseResponse{}, err
+			return TableEntityListByteResponseResponse{}, err
 		}
 		marshalledValue = append(marshalledValue, m)
 	}
@@ -116,7 +117,7 @@ func castToByteResponse(resp *TableEntityQueryResponseResponse) (TableEntityQuer
 		Value:         marshalledValue,
 	}
 
-	return TableEntityQueryByteResponseResponse{
+	return TableEntityListByteResponseResponse{
 		ClientRequestID:                 resp.ClientRequestID,
 		Date:                            resp.Date,
 		RawResponse:                     resp.RawResponse,
@@ -127,3 +128,54 @@ func castToByteResponse(resp *TableEntityQueryResponseResponse) (TableEntityQuer
 		XMSContinuationNextRowKey:       resp.XMSContinuationNextRowKey,
 	}, nil
 }
+
+type TableListResponse struct {
+	// The metadata response of the table.
+	OdataMetadata *string `json:"odata.metadata,omitempty"`
+
+	// List of tables.
+	Value []*TableResponseProperties `json:"value,omitempty"`
+}
+
+func tableListResponseFromQueryResponse(q *TableQueryResponse) *TableListResponse {
+	return &TableListResponse{
+		OdataMetadata: q.OdataMetadata,
+		Value:         q.Value,
+	}
+}
+
+// TableListResponseResponse stores the results of a ListTables operation
+type TableListResponseResponse struct {
+	// ClientRequestID contains the information returned from the x-ms-client-request-id header response.
+	ClientRequestID *string
+
+	// Date contains the information returned from the Date header response.
+	Date *time.Time
+
+	// RawResponse contains the underlying HTTP response.
+	RawResponse *http.Response
+
+	// RequestID contains the information returned from the x-ms-request-id header response.
+	RequestID *string
+
+	// The properties for the table query response.
+	TableListResponse *TableListResponse
+
+	// Version contains the information returned from the x-ms-version header response.
+	Version *string
+
+	// XMSContinuationNextTableName contains the information returned from the x-ms-continuation-NextTableName header response.
+	XMSContinuationNextTableName *string
+}
+
+func listResponseFromQueryResponse(q TableQueryResponseResponse) *TableListResponseResponse {
+	return &TableListResponseResponse{
+		ClientRequestID:              q.ClientRequestID,
+		Date:                         q.Date,
+		RawResponse:                  q.RawResponse,
+		RequestID:                    q.RequestID,
+		TableListResponse:            tableListResponseFromQueryResponse(q.TableQueryResponse),
+		Version:                      q.Version,
+		XMSContinuationNextTableName: q.XMSContinuationNextTableName,
+	}
+}

sdk/tables/aztable/cosmos_patch_transform_policy.go

@@ -11,10 +11,10 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 )
 
-// CosmosPatchTransformPolicy transforms PATCH requests into POST requests with the "X-HTTP-Method":"MERGE" header set.
-type CosmosPatchTransformPolicy struct{}
+// cosmosPatchTransformPolicy transforms PATCH requests into POST requests with the "X-HTTP-Method":"MERGE" header set.
+type cosmosPatchTransformPolicy struct{}
 
-func (p CosmosPatchTransformPolicy) Do(req *azcore.Request) (*azcore.Response, error) {
+func (p cosmosPatchTransformPolicy) Do(req *azcore.Request) (*azcore.Response, error) {
 	transformPatchToCosmosPost(req)
 	return req.Next()
 }

sdk/tables/aztable/entity.go

@@ -13,12 +13,43 @@ import (
 
 // https://docs.microsoft.com/en-us/rest/api/storageservices/payload-format-for-table-service-operations
 
+// The Entity type is the bare minimum properties for a valid Entity. These should be embedded in a custom struct
+// type MyEntity struct {
+// 		Entity
+// 		Value 		int
+// 		StringValue string
+// 		BoolValue 	bool
+// }
+// myEntity := MyEntity{
+// 		Entity: Entity{
+// 			PartitionKey: "pk001",
+// 			RowKey: 	  "rk001",
+// 		},
+// 		Value: 		 10,
+// 		StringValue: "somestring",
+// 		BoolValue: 	 false,
+// }
 type Entity struct {
 	PartitionKey string
 	RowKey       string
 	Timestamp    EdmDateTime
 }
 
+// EdmEntity is an entity that embeds the azcore.Entity type and has a Properties map for an unlimited
+// number of custom properties. The EdmEntity will serialize EdmGuid/EdmInt64/EdmDateTime/EdmBinary according to Odata annotations
+// myEntity := EdmEntity{
+// 		Entity: Entity{
+// 			PartitionKey: "pk001",
+// 			RowKey:       "rk001",
+// 		}
+// 		Properties: map[string]interface{}{
+// 			"Value":    10,
+// 			"Binary":   EdmBinary([]byte{"bytevalue"}),
+// 			"DateTime": EdmDateTime(time.Now()),
+// 			"Int64":    EdmInt64(123456789012345),
+
+// 		}
+// }
 type EdmEntity struct {
 	Metadata string `json:"odata.metadata"`
 	Id       string `json:"odata.id"`
@@ -127,6 +158,8 @@ func (e *EdmEntity) UnmarshalJSON(data []byte) (err error) {
 	return
 }
 
+// EdmBinary represents an Entity Property that is a byte slice. A byte slice wrapped in
+// EdmBinary will also receive the correct odata annotation for round-trip accuracy.
 type EdmBinary []byte
 
 func (e EdmBinary) MarshalText() ([]byte, error) {
@@ -142,6 +175,8 @@ func (e *EdmBinary) UnmarshalText(data []byte) error {
 	return nil
 }
 
+// EdmInt64 represents an entity property that is a 64-bit integer. Using EdmInt64 guarantees
+// proper odata type annotations.
 type EdmInt64 int64
 
 func (e EdmInt64) MarshalText() ([]byte, error) {
@@ -157,6 +192,8 @@ func (e *EdmInt64) UnmarshalText(data []byte) error {
 	return nil
 }
 
+// EdmInt64 represents an entity property that is a GUID wrapped in a string. Using EdmGuid guarantees
+// proper odata type annotations.
 type EdmGuid string
 
 func (e EdmGuid) MarshalText() ([]byte, error) {
@@ -168,6 +205,8 @@ func (e *EdmGuid) UnmarshalText(data []byte) error {
 	return nil
 }
 
+// EdmDateTime represents an entity property that is a time.Time object. Using EdmDateTime guarantees
+// proper odata type annotations.
 type EdmDateTime time.Time
 
 const rfc3339 = "2006-01-02T15:04:05.9999999Z"

sdk/tables/aztable/errors.go

@@ -3,6 +3,10 @@
 
 package aztable
 
+import "errors"
+
+var errInvalidUpdateMode = errors.New("invalid EntityUpdateMode")
+
 func checkEntityForPkRk(entity *map[string]interface{}, err error) error {
 	if _, ok := (*entity)[partitionKey]; !ok {
 		return partitionKeyRowKeyError

sdk/tables/aztable/go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 replace github.com/Azure/azure-sdk-for-go/sdk/internal => ../../internal
 replace github.com/Azure/azure-sdk-for-go/sdk/azidentity => ../../azidentity
+replace github.com/Azure/azure-sdk-for-go/sdk/azcore => ../../azcore
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.16.2

sdk/tables/aztable/models.go

@@ -3,7 +3,7 @@
 
 package aztable
 
-// QueryOptions contains a group of parameters for the Table.Query method.
+// ListOptions contains a group of parameters for the Table.Query method.
 type ListOptions struct {
 	// OData filter expression.
 	Filter *string

sdk/tables/aztable/proxy_test.go

@@ -91,10 +91,10 @@ func createTableServiceClientForRecording(t *testing.T, serviceURL string, cred
 func initClientTest(t *testing.T, service string, createTable bool) (*TableClient, func()) {
 	var client *TableClient
 	var err error
-	if service == string(StorageEndpoint) {
+	if service == string(storageEndpoint) {
 		client, err = createStorageTableClient(t)
 		require.NoError(t, err)
-	} else if service == string(CosmosEndpoint) {
+	} else if service == string(cosmosEndpoint) {
 		client, err = createCosmosTableClient(t)
 		require.NoError(t, err)
 	}
@@ -118,10 +118,10 @@ func initClientTest(t *testing.T, service string, createTable bool) (*TableClien
 func initServiceTest(t *testing.T, service string) (*TableServiceClient, func()) {
 	var client *TableServiceClient
 	var err error
-	if service == string(StorageEndpoint) {
+	if service == string(storageEndpoint) {
 		client, err = createStorageServiceClient(t)
 		require.NoError(t, err)
-	} else if service == string(CosmosEndpoint) {
+	} else if service == string(cosmosEndpoint) {
 		client, err = createCosmosServiceClient(t)
 		require.NoError(t, err)
 	}
@@ -226,7 +226,7 @@ func clearAllTables(service *TableServiceClient) error {
 	pager := service.ListTables(nil)
 	for pager.NextPage(ctx) {
 		resp := pager.PageResponse()
-		for _, v := range resp.TableQueryResponse.Value {
+		for _, v := range resp.TableListResponse.Value {
 			_, err := service.DeleteTable(ctx, *v.TableName, nil)
 			if err != nil {
 				return err