Merge pull request #247 from Azure/dev

Release 10.0.8

ChangeLog.md

@@ -0,0 +1,19 @@
+# Change Log
+
+## Version 10.0.8:
+
+- Rewrote sync command to eliminate numerous bugs and improve usability (see wiki for details)
+- Implemented various improvements to memory management
+- Added MD5 validation support (available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing)
+- Added last modified time checks for source to guarantee transfer integrity 
+- Formalized outputs in JSON and elevated the output flag to the root level
+- Eliminated outputs to STDERR (for new version notifications), which were causing problems for certain CI systems
+- Improved log format for Windows
+- Optimized plan file sizes
+- Improved command line parameter names as follows (to be consistent with naming pattern of other parameters):
+  - fromTo -> from-to
+  - blobType -> blob-type
+  - excludedBlobType -> excluded-blob-type
+  - outputRaw (in "list" command) -> output
+  - stdIn-enable (reserved for internal use) -> stdin-enable
+

Gopkg.toml

@@ -4,7 +4,7 @@
 
 [[constraint]]
   name = "github.com/Azure/azure-storage-blob-go"
-  version = "0.4.0"
+  version = "0.5.0"
 
 [[constraint]]
   name = "github.com/Azure/azure-storage-file-go"

azbfs/url_directory.go

@@ -11,7 +11,7 @@ var directoryResourceName = "directory" // constant value for the resource query
 
 // A DirectoryURL represents a URL to the Azure Storage directory allowing you to manipulate its directories and files.
 type DirectoryURL struct {
-	directoryClient managementClient
+	directoryClient pathClient
 	// filesystem is the filesystem identifier
 	filesystem string
 	// pathParameter is the file or directory path
@@ -24,7 +24,7 @@ func NewDirectoryURL(url url.URL, p pipeline.Pipeline) DirectoryURL {
 		panic("p can't be nil")
 	}
 	urlParts := NewBfsURLParts(url)
-	directoryClient := newManagementClient(url, p)
+	directoryClient := newPathClient(url, p)
 	return DirectoryURL{directoryClient: directoryClient, filesystem: urlParts.FileSystemName, pathParameter: urlParts.DirectoryOrFilePath}
 }
 
@@ -65,8 +65,8 @@ func (d DirectoryURL) NewDirectoryURL(dirName string) DirectoryURL {
 
 // Create creates a new directory within a File System
 func (d DirectoryURL) Create(ctx context.Context) (*DirectoryCreateResponse, error) {
-	resp, err := d.directoryClient.CreatePath(ctx, d.filesystem, d.pathParameter, &directoryResourceName, nil,
-		nil, nil, nil, nil, nil, nil,
+	resp, err := d.directoryClient.Create(ctx, d.filesystem, d.pathParameter, PathResourceDirectory, nil,
+		PathRenameModeNone, nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil, nil,
@@ -77,15 +77,19 @@ func (d DirectoryURL) Create(ctx context.Context) (*DirectoryCreateResponse, err
 // Delete removes the specified empty directory. Note that the directory must be empty before it can be deleted..
 // For more information, see https://docs.microsoft.com/rest/api/storageservices/delete-directory.
 func (d DirectoryURL) Delete(ctx context.Context, continuationString *string, recursive bool) (*DirectoryDeleteResponse, error) {
-	resp, err := d.directoryClient.DeletePath(ctx, d.filesystem, d.pathParameter, &recursive, continuationString, nil,
+	resp, err := d.directoryClient.Delete(ctx, d.filesystem, d.pathParameter, &recursive, continuationString, nil,
 		nil, nil, nil, nil, nil, nil, nil)
 	return (*DirectoryDeleteResponse)(resp), err
 }
 
 // GetProperties returns the directory's metadata and system properties.
 func (d DirectoryURL) GetProperties(ctx context.Context) (*DirectoryGetPropertiesResponse, error) {
-	resp, err := d.directoryClient.GetPathProperties(ctx, d.filesystem, d.pathParameter, nil, nil, nil,
-		nil, nil, nil, nil, nil)
+	// Action MUST be "none", not "getStatus" because the latter does not include the MD5, and
+	// sometimes we call this method on things that are actually files
+	action := PathGetPropertiesActionNone
+
+	resp, err := d.directoryClient.GetProperties(ctx, d.filesystem, d.pathParameter, action, nil, nil,
+		nil, nil, nil, nil, nil, nil, nil)
 	return (*DirectoryGetPropertiesResponse)(resp), err
 }
 
@@ -106,12 +110,13 @@ func (d DirectoryURL) FileSystemURL() FileSystemURL {
 // Marker) to get the next segment.
 func (d DirectoryURL) ListDirectorySegment(ctx context.Context, marker *string, recursive bool) (*DirectoryListResponse, error) {
 	// Since listPath is supported on filesystem Url
-	// covert the directory url to fileSystemUrl
+	// convert the directory url to fileSystemUrl
 	// and listPath for filesystem with directory path set in the path parameter
 	var maxEntriesInListOperation = int32(1000)
 
-	resp, err := d.FileSystemURL().fileSystemClient.ListPaths(ctx, recursive, d.filesystem, fileSystemResourceName, &d.pathParameter, marker,
-		&maxEntriesInListOperation, nil, nil, nil)
+	resp, err := d.FileSystemURL().fileSystemClient.ListPaths(ctx, recursive, d.filesystem, &d.pathParameter, marker,
+		&maxEntriesInListOperation, nil, nil, nil, nil)
+
 	return (*DirectoryListResponse)(resp), err
 }
 

azbfs/url_file.go

@@ -2,17 +2,17 @@ package azbfs
 
 import (
 	"context"
+	"encoding/base64"
 	"net/url"
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
 	"io"
 	"net/http"
-	"strconv"
 )
 
 // A FileURL represents a URL to an Azure Storage file.
 type FileURL struct {
-	fileClient     managementClient
+	fileClient     pathClient
 	fileSystemName string
 	path           string
 }
@@ -22,7 +22,7 @@ func NewFileURL(url url.URL, p pipeline.Pipeline) FileURL {
 	if p == nil {
 		panic("p can't be nil")
 	}
-	fileClient := newManagementClient(url, p)
+	fileClient := newPathClient(url, p)
 
 	urlParts := NewBfsURLParts(url)
 	return FileURL{fileClient: fileClient, fileSystemName: urlParts.FileSystemName, path: urlParts.DirectoryOrFilePath}
@@ -46,10 +46,9 @@ func (f FileURL) WithPipeline(p pipeline.Pipeline) FileURL {
 
 // Create creates a new file or replaces a file. Note that this method only initializes the file.
 // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/create-file.
-func (f FileURL) Create(ctx context.Context) (*CreatePathResponse, error) {
-	fileType := "file"
-	return f.fileClient.CreatePath(ctx, f.fileSystemName, f.path, &fileType,
-		nil, nil, nil, nil, nil, nil,
+func (f FileURL) Create(ctx context.Context) (*PathCreateResponse, error) {
+	return f.fileClient.Create(ctx, f.fileSystemName, f.path, PathResourceFile,
+		nil, PathRenameModeNone, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil,
@@ -62,8 +61,8 @@ func (f FileURL) Create(ctx context.Context) (*CreatePathResponse, error) {
 // response header/property if the range is <= 4MB; the HTTP request fails with 400 (Bad Request) if the requested range is greater than 4MB.
 // For more information, see https://docs.microsoft.com/rest/api/storageservices/get-file.
 func (f FileURL) Download(ctx context.Context, offset int64, count int64) (*DownloadResponse, error) {
-	dr, err := f.fileClient.ReadPath(ctx, f.fileSystemName, f.path, (&httpRange{offset: offset, count: count}).pointers(),
-		nil, nil, nil, nil, nil, nil, nil)
+	dr, err := f.fileClient.Read(ctx, f.fileSystemName, f.path, (&httpRange{offset: offset, count: count}).pointers(),
+		nil, nil, nil, nil, nil, nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -72,7 +71,9 @@ func (f FileURL) Download(ctx context.Context, offset int64, count int64) (*Down
 		f:    f,
 		dr:   dr,
 		ctx:  ctx,
-		info: HTTPGetterInfo{Offset: offset, Count: count, ETag: dr.ETag()}, // TODO: Note conditional header is not currently supported in Azure File.
+		info: HTTPGetterInfo{Offset: offset, Count: count, ETag: dr.ETag()},
+		// TODO: Note conditional header is not currently supported in Azure File.
+		// TODO: review the above todo, since as of 8 Feb 2019 we are on a newer version of the API
 	}, err
 }
 
@@ -99,24 +100,28 @@ func (dr *DownloadResponse) Body(o RetryReaderOptions) io.ReadCloser {
 
 // Delete immediately removes the file from the storage account.
 // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/delete-file2.
-func (f FileURL) Delete(ctx context.Context) (*DeletePathResponse, error) {
+func (f FileURL) Delete(ctx context.Context) (*PathDeleteResponse, error) {
 	recursive := false
-	return f.fileClient.DeletePath(ctx, f.fileSystemName, f.path, &recursive,
+	return f.fileClient.Delete(ctx, f.fileSystemName, f.path, &recursive,
 		nil, nil, nil, nil, nil, nil,
 		nil, nil, nil)
 }
 
 // GetProperties returns the file's metadata and properties.
 // For more information, see https://docs.microsoft.com/rest/api/storageservices/get-file-properties.
-func (f FileURL) GetProperties(ctx context.Context) (*GetPathPropertiesResponse, error) {
-	return f.fileClient.GetPathProperties(ctx, f.fileSystemName, f.path, nil, nil,
+func (f FileURL) GetProperties(ctx context.Context) (*PathGetPropertiesResponse, error) {
+	// Action MUST be "none", not "getStatus" because the latter does not include the MD5, and
+	// sometimes we call this method on things that are actually files
+	action := PathGetPropertiesActionNone
+
+	return f.fileClient.GetProperties(ctx, f.fileSystemName, f.path, action, nil,
 		nil, nil, nil,
-		nil, nil, nil)
+		nil, nil, nil, nil, nil)
 }
 
 // UploadRange writes bytes to a file.
 // offset indiciates the offset at which to begin writing, in bytes.
-func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeker) (*UpdatePathResponse, error) {
+func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeker) (*PathUpdateResponse, error) {
 	if offset < 0 {
 		panic("offset must be >= 0")
 	}
@@ -128,21 +133,29 @@ func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeke
 	if count == 0 {
 		panic("body must contain readable data whose size is > 0")
 	}
-	countAsStr := strconv.FormatInt(count, 10)
 
-	// TODO the go http client has a problem with PATCH and content-length header
-	// TODO we should investigate and report the issue
+	// TODO: the go http client has a problem with PATCH and content-length header
+	//                we should investigate and report the issue
+	// Note: the "offending" code in the Go SDK is: func (t *transferWriter) shouldSendContentLength() bool
+	// That code suggests that a workaround would be to specify a Transfer-Encoding of "identity",
+	// but we haven't yet found any way to actually set that header, so that workaround does't
+	// seem to work. (Just setting Transfer-Encoding like a normal header doesn't seem to work.)
+	// Looks like it might actually be impossible to set
+	// the Transfer-Encoding header, because bradfitz wrote: "as a general rule of thumb, you don't get to mess
+	// with [that field] too much. The net/http package owns much of its behavior."
+	// https://grokbase.com/t/gg/golang-nuts/15bg66ryd9/go-nuts-cant-write-encoding-other-than-chunked-in-the-transfer-encoding-field-of-http-request
 	overrideHttpVerb := "PATCH"
 
 	// TransactionalContentMD5 isn't supported currently.
-	return f.fileClient.UpdatePath(ctx, "append", f.fileSystemName, f.path, &offset,
-		nil, &countAsStr, nil, nil, nil, nil,
+	return f.fileClient.Update(ctx, PathUpdateActionAppend, f.fileSystemName, f.path, &offset,
+		nil, nil, nil, nil, nil, nil,
 		nil, nil, nil, nil, nil, nil, nil,
-		nil, nil, nil, nil, nil, &overrideHttpVerb, body, nil, nil, nil)
+		nil, nil, nil, nil, nil, nil, &overrideHttpVerb, body, nil, nil, nil)
 }
 
 // flushes writes previously uploaded data to a file
-func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*UpdatePathResponse, error) {
+// The contentMd5 parameter, if not nil, should represent the MD5 hash that has been computed for the file as whole
+func (f FileURL) FlushData(ctx context.Context, fileSize int64, contentMd5 []byte) (*PathUpdateResponse, error) {
 	if fileSize < 0 {
 		panic("fileSize must be >= 0")
 	}
@@ -151,14 +164,24 @@ func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*UpdatePathResp
 	// azcopy does not need this
 	retainUncommittedData := false
 
-	// TODO the go http client has a problem with PATCH and content-length header
-	// TODO we should investigate and report the issue
+	var md5InBase64 *string = nil
+	if len(contentMd5) > 0 {
+		enc := base64.StdEncoding.EncodeToString(contentMd5)
+		md5InBase64 = &enc
+	}
+
+	// TODO: the go http client has a problem with PATCH and content-length header
+	//       we should investigate and report the issue
+	// See similar todo, with larger comments, in AppendData
 	overrideHttpVerb := "PATCH"
 
+	// TODO: feb 2019 API update: review the use of closeParameter here. Should it be true?
+	//    Doc implies only make it true if this is the end of the file
+
 	// TransactionalContentMD5 isn't supported currently.
-	return f.fileClient.UpdatePath(ctx, "flush", f.fileSystemName, f.path, &fileSize,
+	return f.fileClient.Update(ctx, PathUpdateActionFlush, f.fileSystemName, f.path, &fileSize,
 		&retainUncommittedData, nil, nil, nil, nil, nil,
-		nil, nil, nil, nil, nil,
+		nil, nil, nil, md5InBase64, nil,
 		nil, nil, nil, nil, nil, nil, nil,
-		&overrideHttpVerb, nil, nil, nil, nil)
+		nil, &overrideHttpVerb, nil, nil, nil, nil)
 }

azbfs/url_filesystem.go

@@ -7,11 +7,9 @@ import (
 	"github.com/Azure/azure-pipeline-go/pipeline"
 )
 
-const fileSystemResourceName = "filesystem" // constant value for the resource query parameter
-
 // A FileSystemURL represents a URL to the Azure Storage Blob File System allowing you to manipulate its directories and files.
 type FileSystemURL struct {
-	fileSystemClient managementClient
+	fileSystemClient filesystemClient
 	name             string
 }
 
@@ -20,7 +18,7 @@ func NewFileSystemURL(url url.URL, p pipeline.Pipeline) FileSystemURL {
 	if p == nil {
 		panic("p can't be nil")
 	}
-	fileSystemClient := newManagementClient(url, p)
+	fileSystemClient := newFilesystemClient(url, p)
 
 	urlParts := NewBfsURLParts(url)
 	return FileSystemURL{fileSystemClient: fileSystemClient, name: urlParts.FileSystemName}
@@ -62,17 +60,17 @@ func (s FileSystemURL) NewRootDirectoryURL() DirectoryURL {
 
 // Create creates a new file system within a storage account. If a file system with the same name already exists, the operation fails.
 // quotaInGB specifies the maximum size of the file system in gigabytes, 0 means you accept service's default quota.
-func (s FileSystemURL) Create(ctx context.Context) (*CreateFilesystemResponse, error) {
-	return s.fileSystemClient.CreateFilesystem(ctx, s.name, fileSystemResourceName, nil, nil, nil, nil)
+func (s FileSystemURL) Create(ctx context.Context) (*FilesystemCreateResponse, error) {
+	return s.fileSystemClient.Create(ctx, s.name, nil, nil, nil, nil)
 }
 
 // Delete marks the specified file system for deletion.
 // The file system and any files contained within it are later deleted during garbage collection.
-func (s FileSystemURL) Delete(ctx context.Context) (*DeleteFilesystemResponse, error) {
-	return s.fileSystemClient.DeleteFilesystem(ctx, s.name, fileSystemResourceName, nil, nil, nil, nil, nil)
+func (s FileSystemURL) Delete(ctx context.Context) (*FilesystemDeleteResponse, error) {
+	return s.fileSystemClient.Delete(ctx, s.name, nil, nil, nil, nil, nil)
 }
 
 // GetProperties returns all user-defined metadata and system properties for the specified file system or file system snapshot.
-func (s FileSystemURL) GetProperties(ctx context.Context) (*GetFilesystemPropertiesResponse, error) {
-	return s.fileSystemClient.GetFilesystemProperties(ctx, s.name, fileSystemResourceName, nil, nil, nil)
+func (s FileSystemURL) GetProperties(ctx context.Context) (*FilesystemGetPropertiesResponse, error) {
+	return s.fileSystemClient.GetProperties(ctx, s.name, nil, nil, nil)
 }