*: Port to new Azure SDK

go.mod

@@ -3,11 +3,21 @@ module github.com/flatcar/azure-vhd-utils
 go 1.20
 
 require (
-	github.com/Azure/azure-sdk-for-go v5.0.0-beta.0.20161118192335-3b1282355199+incompatible
-	gopkg.in/urfave/cli.v1 v1.19.1
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2
+	gopkg.in/urfave/cli.v1 v1.20.0
 )
 
 require (
-	github.com/kr/pretty v0.1.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 )

go.sum

@@ -1,11 +1,34 @@
-github.com/Azure/azure-sdk-for-go v5.0.0-beta.0.20161118192335-3b1282355199+incompatible h1:onD5f6jmuNCkgou4duYKTDFBxIJo0PRLyL3RgJkIt3Q=
-github.com/Azure/azure-sdk-for-go v5.0.0-beta.0.20161118192335-3b1282355199+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/urfave/cli.v1 v1.19.1 h1:pkwzWQSFerxgLtkdWlnjwOS+Vd7VCp/Kwdn3kmeflXQ=
-gopkg.in/urfave/cli.v1 v1.19.1/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
+gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

upload/metadata/metaData.go → upload/metadata/metadata.go

@@ -10,58 +10,56 @@ import (
 	"os"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/storage"
-
 	"github.com/flatcar/azure-vhd-utils/upload/progress"
 	"github.com/flatcar/azure-vhd-utils/vhdcore/diskstream"
 )
 
 // The key of the page blob metadata collection entry holding VHD metadata as json.
-const metaDataKey = "diskmetadata"
+const metadataKey = "diskmetadata"
 
-// MetaData is the type representing metadata associated with an Azure page blob holding the VHD.
+// Metadata is the type representing metadata associated with an Azure page blob holding the VHD.
 // This will be stored as a JSON string in the page blob metadata collection with key 'diskmetadata'.
-type MetaData struct {
-	FileMetaData *FileMetaData `json:"fileMetaData"`
+type Metadata struct {
+	FileMetadata *FileMetadata `json:"fileMetaData"`
 }
 
-// FileMetaData represents the metadata of a VHD file.
-type FileMetaData struct {
+// FileMetadata represents the metadata of a VHD file.
+type FileMetadata struct {
 	FileName         string    `json:"fileName"`
 	FileSize         int64     `json:"fileSize"`
 	VHDSize          int64     `json:"vhdSize"`
 	LastModifiedTime time.Time `json:"lastModifiedTime"`
 	MD5Hash          []byte    `json:"md5Hash"` // Marshal will encodes []byte as a base64-encoded string
 }
 
-// ToJSON returns MetaData as a json string.
-func (m *MetaData) ToJSON() (string, error) {
+// ToJSON returns Metadata as a json string.
+func (m *Metadata) ToJSON() (string, error) {
 	b, err := json.Marshal(m)
 	if err != nil {
 		return "", err
 	}
 	return string(b), nil
 }
 
-// ToMap returns the map representation of the MetaData which can be stored in the page blob metadata colleciton
-func (m *MetaData) ToMap() (map[string]string, error) {
+// ToMap returns the map representation of the Metadata which can be stored in the page blob metadata collection.
+func (m *Metadata) ToMap() (map[string]*string, error) {
 	v, err := m.ToJSON()
 	if err != nil {
 		return nil, err
 	}
 
-	return map[string]string{metaDataKey: v}, nil
+	return map[string]*string{metadataKey: &v}, nil
 }
 
-// NewMetaDataFromLocalVHD creates a MetaData instance that should be associated with the page blob
+// NewMetadataFromLocalVHD creates a Metadata instance that should be associated with the page blob
 // holding the VHD. The parameter vhdPath is the path to the local VHD.
-func NewMetaDataFromLocalVHD(vhdPath string) (*MetaData, error) {
+func NewMetadataFromLocalVHD(vhdPath string) (*Metadata, error) {
 	fileStat, err := getFileStat(vhdPath)
 	if err != nil {
 		return nil, err
 	}
 
-	fileMetaData := &FileMetaData{
+	fileMetadata := &FileMetadata{
 		FileName:         fileStat.Name(),
 		FileSize:         fileStat.Size(),
 		LastModifiedTime: fileStat.ModTime(),
@@ -72,71 +70,65 @@ func NewMetaDataFromLocalVHD(vhdPath string) (*MetaData, error) {
 		return nil, err
 	}
 	defer diskStream.Close()
-	fileMetaData.VHDSize = diskStream.GetSize()
-	fileMetaData.MD5Hash, err = calculateMD5Hash(diskStream)
+	fileMetadata.VHDSize = diskStream.GetSize()
+	fileMetadata.MD5Hash, err = calculateMD5Hash(diskStream)
 	if err != nil {
 		return nil, err
 	}
 
-	return &MetaData{
-		FileMetaData: fileMetaData,
+	return &Metadata{
+		FileMetadata: fileMetadata,
 	}, nil
 }
 
-// NewMetadataFromBlob returns MetaData instance associated with a Azure page blob, if there is no
-// MetaData associated with the blob it returns nil value for MetaData
-func NewMetadataFromBlob(blobClient storage.BlobStorageClient, containerName, blobName string) (*MetaData, error) {
-	allMetadata, err := blobClient.GetBlobMetadata(containerName, blobName)
-	if err != nil {
-		return nil, fmt.Errorf("NewMetadataFromBlob, failed to fetch blob metadata: %v", err)
-	}
-	m, ok := allMetadata[metaDataKey]
-	if !ok {
+// NewMetadataFromBlobMetadata returns Metadata instance associated with a Azure page blob, if there is no Metadata
+// associated with the blob it returns nil value for Metadata
+func NewMetadataFromBlobMetadata(blobmd map[string]*string) (*Metadata, error) {
+	m, ok := blobmd[metadataKey]
+	if !ok || m == nil {
 		return nil, nil
 	}
-
-	b := []byte(m)
-	metadata := MetaData{}
-	if err := json.Unmarshal(b, &metadata); err != nil {
-		return nil, fmt.Errorf("NewMetadataFromBlob, failed to deserialize blob metadata with key %s: %v", metaDataKey, err)
+	metadata := new(Metadata)
+	if err := json.Unmarshal([]byte(*m), metadata); err != nil {
+		return nil, fmt.Errorf("NewMetadataFromBlobMetadata, failed to deserialize blob metadata with key %s: %v", metadataKey, err)
 	}
-	return &metadata, nil
+	return metadata, nil
 }
 
-// CompareMetaData compares the MetaData associated with the remote page blob and local VHD file. If both metadata
+// CompareMetadata compares the Metadata associated with the remote page blob and local VHD file. If both metadata
 // are same this method returns an empty error slice else a non-empty error slice with each error describing
 // the metadata entry that mismatched.
-func CompareMetaData(remote, local *MetaData) []error {
+func CompareMetadata(remote, local *Metadata) []error {
 	var metadataErrors = make([]error, 0)
-	if !bytes.Equal(remote.FileMetaData.MD5Hash, local.FileMetaData.MD5Hash) {
+	if !bytes.Equal(remote.FileMetadata.MD5Hash, local.FileMetadata.MD5Hash) {
 		metadataErrors = append(metadataErrors,
 			fmt.Errorf("MD5 hash of VHD file in Azure blob storage (%v) and local VHD file (%v) does not match",
-				base64.StdEncoding.EncodeToString(remote.FileMetaData.MD5Hash),
-				base64.StdEncoding.EncodeToString(local.FileMetaData.MD5Hash)))
+				base64.StdEncoding.EncodeToString(remote.FileMetadata.MD5Hash),
+				base64.StdEncoding.EncodeToString(local.FileMetadata.MD5Hash)))
 	}
 
-	if remote.FileMetaData.VHDSize != local.FileMetaData.VHDSize {
+	if remote.FileMetadata.VHDSize != local.FileMetadata.VHDSize {
 		metadataErrors = append(metadataErrors,
 			fmt.Errorf("Logical size of the VHD file in Azure blob storage (%d) and local VHD file (%d) does not match",
-				remote.FileMetaData.VHDSize, local.FileMetaData.VHDSize))
+				remote.FileMetadata.VHDSize, local.FileMetadata.VHDSize))
 	}
 
-	if remote.FileMetaData.FileSize != local.FileMetaData.FileSize {
+	if remote.FileMetadata.FileSize != local.FileMetadata.FileSize {
 		metadataErrors = append(metadataErrors,
 			fmt.Errorf("Size of the VHD file in Azure blob storage (%d) and local VHD file (%d) does not match",
-				remote.FileMetaData.FileSize, local.FileMetaData.FileSize))
+				remote.FileMetadata.FileSize, local.FileMetadata.FileSize))
 	}
 
-	if remote.FileMetaData.LastModifiedTime != local.FileMetaData.LastModifiedTime {
+	if remote.FileMetadata.LastModifiedTime != local.FileMetadata.LastModifiedTime {
 		metadataErrors = append(metadataErrors,
 			fmt.Errorf("Last modified time of the VHD file in Azure blob storage (%v) and local VHD file (%v) does not match",
-				remote.FileMetaData.LastModifiedTime, local.FileMetaData.LastModifiedTime))
+				remote.FileMetadata.LastModifiedTime, local.FileMetadata.LastModifiedTime))
 	}
 
-	if remote.FileMetaData.FileName != local.FileMetaData.FileName {
+	if remote.FileMetadata.FileName != local.FileMetadata.FileName {
 		metadataErrors = append(metadataErrors,
 			fmt.Errorf("Full name of the VHD file in Azure blob storage (%s) and local VHD file (%s) does not match",
-				remote.FileMetaData.FileName, local.FileMetaData.FileName))
+				remote.FileMetadata.FileName, local.FileMetadata.FileName))
 	}
 
 	return metadataErrors
@@ -146,7 +138,7 @@ func CompareMetaData(remote, local *MetaData) []error {
 func getFileStat(filePath string) (os.FileInfo, error) {
 	fd, err := os.Open(filePath)
 	if err != nil {
-		return nil, fmt.Errorf("fileMetaData.getFileStat: %v", err)
+		return nil, fmt.Errorf("fileMetadata.getFileStat: %v", err)
 	}
 	defer fd.Close()
 	return fd.Stat()

upload/upload.go

@@ -1,37 +1,51 @@
 package upload
 
 import (
+	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/pageblob"
 
 	"github.com/flatcar/azure-vhd-utils/upload/concurrent"
 	"github.com/flatcar/azure-vhd-utils/upload/progress"
 	"github.com/flatcar/azure-vhd-utils/vhdcore/common"
 	"github.com/flatcar/azure-vhd-utils/vhdcore/diskstream"
 )
 
-// DiskUploadContext type describes VHD upload context, this includes the disk stream to read from, the ranges of
-// the stream to read, the destination blob and it's container, the client to communicate with Azure storage and
-// the number of parallel go-routines to use for upload.
+// DiskUploadContext type describes VHD upload context, this includes the disk stream to read from, the ranges of the
+// stream to read, the client representing the destination blob in its container and used to communicate with Azure
+// storage and the number of parallel go-routines to use for upload.
 type DiskUploadContext struct {
-	VhdStream             *diskstream.DiskStream    // The stream whose ranges needs to be uploaded
-	AlreadyProcessedBytes int64                     // The size in bytes already uploaded
-	UploadableRanges      []*common.IndexRange      // The subset of stream ranges to be uploaded
-	BlobServiceClient     storage.BlobStorageClient // The client to make Azure blob service API calls
-	ContainerName         string                    // The container in which page blob resides
-	BlobName              string                    // The destination page blob name
-	Parallelism           int                       // The number of concurrent goroutines to be used for upload
-	Resume                bool                      // Indicate whether this is a new or resuming upload
-	MD5Hash               []byte                    // MD5Hash to be set in the page blob properties once upload finishes
+	VhdStream             *diskstream.DiskStream // The stream whose ranges needs to be uploaded
+	AlreadyProcessedBytes int64                  // The size in bytes already uploaded
+	UploadableRanges      []*common.IndexRange   // The subset of stream ranges to be uploaded
+	PageblobClient        *pageblob.Client       // The client to make Azure blob service API calls
+	Parallelism           int                    // The number of concurrent goroutines to be used for upload
+	Resume                bool                   // Indicate whether this is a new or resuming upload
 }
 
 // oneMB is one MegaByte
 const oneMB = float64(1048576)
 
+type byteReadSeekCloser struct {
+	*bytes.Reader
+}
+
+func newByteReadSeekCloser(b []byte) io.ReadSeekCloser {
+	return byteReadSeekCloser{bytes.NewReader(b)}
+}
+
+func (byteReadSeekCloser) Close() error {
+	return nil
+}
+
+var _ io.ReadSeekCloser = byteReadSeekCloser{}
+
 // Upload uploads the disk ranges described by the parameter cxt, this parameter describes the disk stream to
 // read from, the ranges of the stream to read, the destination blob and it's container, the client to communicate
 // with Azure storage and the number of parallel go-routines to use for upload.
@@ -84,12 +98,12 @@ L:
 			//
 			req := &concurrent.Request{
 				Work: func() error {
-					err := cxt.BlobServiceClient.PutPage(cxt.ContainerName,
-						cxt.BlobName,
-						dataWithRange.Range.Start,
-						dataWithRange.Range.End,
-						storage.PageWriteTypeUpdate,
-						dataWithRange.Data,
+					_, err := cxt.PageblobClient.UploadPages(context.TODO(),
+						newByteReadSeekCloser(dataWithRange.Data),
+						blob.HTTPRange{
+							Offset: dataWithRange.Range.Start,
+							Count:  dataWithRange.Range.Length(),
+						},
 						nil)
 					if err == nil {
 						uploadProgress.ReportBytesProcessedCount(dataWithRange.Range.Length())

upload/uploadableRanges.go

@@ -5,15 +5,17 @@ import (
 	"github.com/flatcar/azure-vhd-utils/vhdcore/diskstream"
 )
 
-// LocateUploadableRanges detects the uploadable ranges in a VHD stream, size of each range is at most pageSizeInBytes.
+// LocateUploadableRanges detects the uploadable ranges in a VHD stream, size of each range is at most
+// pageSetSizeInBytes and a multiple of pageSizeInBytes (in other words, each range represents a set of pageSizeInBytes-sized
+// pages that together take at most pageSetSizeInBytes bytes).
 //
 // This method reads the existing ranges A from the disk stream, creates a new set of ranges B from A by removing the
 // ranges identified by the parameter rangesToSkip, returns new set of ranges C (with each range of size at most
-// pageSizeInBytes) by merging adjacent ranges in B or splitting ranges in B.
+// pageSetSizeInBytes) by merging adjacent ranges in B or splitting ranges in B.
 //
 // Note that this method will not check whether ranges of a fixed disk contains zeros, hence inorder to filter out such
 // ranges from the uploadable ranges, caller must use LocateNonEmptyRangeIndices method.
-func LocateUploadableRanges(stream *diskstream.DiskStream, rangesToSkip []*common.IndexRange, pageSizeInBytes int64) ([]*common.IndexRange, error) {
+func LocateUploadableRanges(stream *diskstream.DiskStream, rangesToSkip []*common.IndexRange, pageSizeInBytes, pageSetSizeInBytes int64) ([]*common.IndexRange, error) {
 	var err error
 	var diskRanges = make([]*common.IndexRange, 0)
 	stream.EnumerateExtents(func(ext *diskstream.StreamExtent, extErr error) bool {
@@ -31,6 +33,6 @@ func LocateUploadableRanges(stream *diskstream.DiskStream, rangesToSkip []*commo
 	}
 
 	diskRanges = common.SubtractRanges(diskRanges, rangesToSkip)
-	diskRanges = common.ChunkRangesBySize(diskRanges, pageSizeInBytes)
+	diskRanges = common.ChunkRangesBySizeWithQuant(diskRanges, pageSetSizeInBytes, pageSizeInBytes)
 	return diskRanges, nil
 }