From a82673434906b9c9028fd4cff61011592dec18cf Mon Sep 17 00:00:00 2001 From: AK Date: Wed, 30 Jan 2019 16:09:51 -0800 Subject: [PATCH 01/64] Updating the help messages for AzCopyV10 --- cmd/helpMessages.go | 80 ++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/cmd/helpMessages.go b/cmd/helpMessages.go index ec796f776..89fba0568 100644 --- a/cmd/helpMessages.go +++ b/cmd/helpMessages.go @@ -28,9 +28,9 @@ Copies source data to a destination location. The supported pairs are: Please refer to the examples for more information. Advanced: -Please note that AzCopy automatically detects the Content-Type of files when uploading from local disk, based on file extension or file content(if no extension). +Please note that AzCopy automatically detects the Content Type of the files when uploading from the local disk, based on the file extension or content (if no extension is specified). -The built-in lookup table is small but on unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: +The built-in lookup table is small but on Unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: - /etc/mime.types - /etc/apache2/mime.types - /etc/apache/mime.types @@ -38,52 +38,52 @@ The built-in lookup table is small but on unix it is augmented by the local syst On Windows, MIME types are extracted from the registry. This feature can be turned off with the help of a flag. Please refer to the flag section. ` -const copyCmdExample = `Upload a single file with SAS: - - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" +const copyCmdExample = `Upload a single file using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet: +- azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" -Upload a single file with OAuth token, please use login command first if not yet logged in: - - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" +Upload a single file with a SAS: + - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Upload a single file through piping(block blob only) with SAS: +Upload a single file with a SAS using a pipeline (block blobs only): - cat "/path/to/file.txt" | azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Upload an entire directory with SAS: +Upload an entire directory with a SAS: - azcopy cp "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Upload only files using wildcards with SAS: +Upload a set of files with a SAS using wildcards: - azcopy cp "/path/*foo/*bar/*.pdf" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" -Upload files and directories using wildcards with SAS: +Upload files and directories with a SAS using wildcards: - azcopy cp "/path/*foo/*bar*" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Download a single file with SAS: - - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "/path/to/file.txt" - -Download a single file with OAuth token, please use login command first if not yet logged in: +Download a single file using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet: - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]" "/path/to/file.txt" -Download a single file through piping(blobs only) with SAS: +Download a single file with a SAS: + - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "/path/to/file.txt" + +Download a single file with a SAS using a pipeline (block blobs only): - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" > "/path/to/file.txt" -Download an entire directory with SAS: +Download an entire directory with a SAS: - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" "/path/to/dir" --recursive=true -Download files using wildcards with SAS: +Download a set of files with a SAS using wildcards: - azcopy cp "https://[account].blob.core.windows.net/[container]/foo*?[SAS]" "/path/to/dir" -Download files and directories using wildcards with SAS: +Download files and directories with a SAS using wildcards: - azcopy cp "https://[account].blob.core.windows.net/[container]/foo*?[SAS]" "/path/to/dir" --recursive=true -Copy a single file with SAS: +Copy a single file between two storage accounts with a SAS: - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Copy a single file with OAuth token, please use login command first if not yet logged in and note that OAuth token is used by destination: +Copy a single file between two storage accounts using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet. Note that the same OAuth token is used to access the destination storage account: - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]" -Copy an entire directory with SAS: +Copy an entire directory between two storage accounts with a SAS: - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Copy an entire account with SAS: +Copy an entire account data to another account with SAS: - azcopy cp "https://[srcaccount].blob.core.windows.net?[SAS]" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true ` @@ -108,7 +108,7 @@ Display information on all jobs.` const showJobsCmdShortDescription = "Show detailed information for the given job ID" const showJobsCmdLongDescription = ` -Show detailed information for the given job ID: if only the job ID is supplied without flag, then the progress summary of the job is returned. +Show detailed information for the given job ID: if only the job ID is supplied without a flag, then the progress summary of the job is returned. If the with-status flag is set, then the list of transfers in the job with the given value will be shown.` const resumeJobsCmdShortDescription = "Resume the existing job with the given job ID" @@ -124,38 +124,38 @@ const listCmdLongDescription = `List the entities in a given resource. Only Blob const listCmdExample = "azcopy list [containerURL]" // ===================================== LOGIN COMMAND ===================================== // -const loginCmdShortDescription = "Log in to Azure Active Directory to access Azure storage resources." +const loginCmdShortDescription = "Log in to Azure Active Directory to access Azure Storage resources." -const loginCmdLongDescription = `Log in to Azure Active Directory to access Azure storage resources. +const loginCmdLongDescription = `Log in to Azure Active Directory to access Azure Storage resources. Note that, to be authorized to your Azure Storage account, you must assign your user 'Storage Blob Data Contributor' role on the Storage account. -This command will cache encrypted login info for current user with OS built-in mechanisms. +This command will cache encrypted login information for current user using the OS built-in mechanisms. Please refer to the examples for more information.` const loginCmdExample = `Log in interactively with default AAD tenant ID set to common: - azcopy login -Log in interactively with specified tenant ID: +Log in interactively with a specified tenant ID: - azcopy login --tenant-id "[TenantID]" Log in using a VM's system-assigned identity: - azcopy login --identity -Log in using a VM's user-assigned identity with Client ID of the service identity: +Log in using a VM's user-assigned identity with a Client ID of the service identity: - azcopy login --identity --identity-client-id "[ServiceIdentityClientID]" -Log in using a VM's user-assigned identity with Object ID of the service identity: +Log in using a VM's user-assigned identity with an Object ID of the service identity: - azcopy login --identity --identity-object-id "[ServiceIdentityObjectID]" -Log in using a VM's user-assigned identity with Resource ID of the service identity: +Log in using a VM's user-assigned identity with a Resource ID of the service identity: - azcopy login --identity --identity-resource-id "/subscriptions//resourcegroups/myRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myID" ` // ===================================== LOGOUT COMMAND ===================================== // -const logoutCmdShortDescription = "Log out to remove access to Azure storage resources." +const logoutCmdShortDescription = "Log out to terminate access to Azure Storage resources." -const logoutCmdLongDescription = `Log out to remove access to Azure storage resources. -This command will remove all the cached login info for current user.` +const logoutCmdLongDescription = `Log out to terminate access to Azure Storage resources. +This command will remove all the cached login information for the current user.` // ===================================== MAKE COMMAND ===================================== // const makeCmdShortDescription = "Create a container/share/filesystem" @@ -167,21 +167,21 @@ const makeCmdExample = ` ` // ===================================== REMOVE COMMAND ===================================== // -const removeCmdShortDescription = "Deletes blobs or files in Azure Storage" +const removeCmdShortDescription = "Delete blobs or files from Azure Storage" -const removeCmdLongDescription = `Deletes blobs or files in Azure Storage.` +const removeCmdLongDescription = `Delete blobs or files from Azure Storage.` // ===================================== SYNC COMMAND ===================================== // -const syncCmdShortDescription = "Replicates source to the destination location" +const syncCmdShortDescription = "Replicate source to the destination location" const syncCmdLongDescription = ` -Replicates source to the destination location. The last modified times are used for comparison. The supported pairs are: - - local <-> Azure Blob (SAS or OAuth authentication) +Replicate a source to a destination location. The last modified times are used for comparison, the file is skipped if the last modified time in the destination is more recent. The supported pairs are: + - local <-> Azure Blob (either SAS or OAuth authentication can be used) Advanced: -Please note that AzCopy automatically detects the Content-Type of files when uploading from local disk, based on file extension or file content(if no extension). +Please note that AzCopy automatically detects the Content Type of the files when uploading from the local disk, based on the file extension or content (if no extension is specified). -The built-in lookup table is small but on unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: +The built-in lookup table is small but on Unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: - /etc/mime.types - /etc/apache2/mime.types - /etc/apache/mime.types From ddf4ed2ed6759a478ffda935497ef2a7c351914c Mon Sep 17 00:00:00 2001 From: JohnRusk <25887678+JohnRusk@users.noreply.github.com> Date: Mon, 4 Feb 2019 12:10:32 +1300 Subject: [PATCH 02/64] Fix/enable body read retries and logging (#194) * Port forced retry and better logging to the azbfs retry reader (Same as already in Blob/File SDKs) * Enable logging of retries in response body downloads * Force body read retries when a downloaded chunk is abnormally slow Without this, we can see very occasional chunks that are more than an order of magnitude slower than normal. That becomes a problem given our desire to process the files sequentially (for MD5 computation and, in some cases, disk performance). Root cause of the slowness is yet to be determined, but this workaround is effective for now. * Update "dep" dependency of blob SDK to latest version --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- azbfs/zc_retry_reader.go | 89 +++++++-- azbfs/zt_retry_reader_test.go | 330 ++++++++++++++++++++++++++++++++++ common/chunkedFileWriter.go | 22 ++- common/logger.go | 30 ++++ ste/downloader-azureFiles.go | 14 +- ste/downloader-blob.go | 11 +- ste/downloader-blobFS.go | 12 +- 9 files changed, 472 insertions(+), 44 deletions(-) create mode 100644 azbfs/zt_retry_reader_test.go diff --git a/Gopkg.lock b/Gopkg.lock index ad3367d91..6949845dd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -18,12 +18,12 @@ version = "v0.1.8" [[projects]] - digest = "1:b8ac7e4464ce21f7487c663aa69b1b3437740bb10ab12d4dc7aa9b02422571a1" + digest = "1:435043934aa0a8221e2c660e88dffe588783e9497facf6b517a465a37c58c97b" name = "github.com/Azure/azure-storage-blob-go" packages = ["azblob"] pruneopts = "UT" - revision = "45d0c5e3638e2b539942f93c48e419f4f2fc62e4" - version = "0.4.0" + revision = "457680cc0804810f6d02958481e0ffdda51d5c60" + version = "0.5.0" [[projects]] digest = "1:0376b54cf965bdae03b9c2a2734cad87fc3bbaa4066c0374ec86f10f77f03d45" diff --git a/Gopkg.toml b/Gopkg.toml index e9a5f06f7..722aeb665 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -4,7 +4,7 @@ [[constraint]] name = "github.com/Azure/azure-storage-blob-go" - version = "0.4.0" + version = "0.5.0" [[constraint]] name = "github.com/Azure/azure-storage-file-go" diff --git a/azbfs/zc_retry_reader.go b/azbfs/zc_retry_reader.go index 988566745..05cb79b64 100644 --- a/azbfs/zc_retry_reader.go +++ b/azbfs/zc_retry_reader.go @@ -5,6 +5,8 @@ import ( "io" "net" "net/http" + "strings" + "sync" ) // HTTPGetter is a function type that refers to a method that performs an HTTP GET operation. @@ -26,6 +28,9 @@ type HTTPGetterInfo struct { ETag string } +// FailedReadNotifier is a function type that represents the notification function called when a read fails +type FailedReadNotifier func(failureCount int, lastError error, offset int64, count int64, willRetry bool) + // RetryReaderOptions contains properties which can help to decide when to do retry. type RetryReaderOptions struct { // MaxRetryRequests specifies the maximum number of HTTP GET requests that will be made @@ -34,6 +39,20 @@ type RetryReaderOptions struct { MaxRetryRequests int doInjectError bool doInjectErrorRound int + + // NotifyFailedRead is called, if non-nil, after any failure to read. Expected usage is diagnostic logging. + NotifyFailedRead FailedReadNotifier + + // TreatEarlyCloseAsError can be set to true to prevent retries after "read on closed response body". By default, + // retryReader has the following special behaviour: closing the response body before it is all read is treated as a + // retryable error. This is to allow callers to force a retry by closing the body from another goroutine (e.g. if the = + // read is too slow, caller may want to force a retry in the hope that the retry will be quicker). If + // TreatEarlyCloseAsError is true, then retryReader's special behaviour is suppressed, and "read on closed body" is instead + // treated as a fatal (non-retryable) error. + // Note that setting TreatEarlyCloseAsError only guarantees that Closing will produce a fatal error if the Close happens + // from the same "thread" (goroutine) as Read. Concurrent Close calls from other goroutines may instead produce network errors + // which will be retried. + TreatEarlyCloseAsError bool } // retryReader implements io.ReaderCloser methods. @@ -43,11 +62,14 @@ type RetryReaderOptions struct { // through reading from the new response. type retryReader struct { ctx context.Context - response *http.Response info HTTPGetterInfo countWasBounded bool o RetryReaderOptions getter HTTPGetter + + // we support Close-ing during Reads (from other goroutines), so we protect the shared state, which is response + responseMu *sync.Mutex + response *http.Response } // NewRetryReader creates a retry reader. @@ -62,7 +84,20 @@ func NewRetryReader(ctx context.Context, initialResponse *http.Response, if o.MaxRetryRequests < 0 { panic("o.MaxRetryRequests must be >= 0") } - return &retryReader{ctx: ctx, getter: getter, info: info, countWasBounded: info.Count != CountToEnd, response: initialResponse, o: o} + return &retryReader{ + ctx: ctx, + getter: getter, + info: info, + countWasBounded: info.Count != CountToEnd, + response: initialResponse, + responseMu: &sync.Mutex{}, + o: o} +} + +func (s *retryReader) setResponse(r *http.Response) { + s.responseMu.Lock() + defer s.responseMu.Unlock() + s.response = r } func (s *retryReader) Read(p []byte) (n int, err error) { @@ -73,15 +108,19 @@ func (s *retryReader) Read(p []byte) (n int, err error) { return 0, io.EOF } - if s.response == nil { // We don't have a response stream to read from, try to get one. - response, err := s.getter(s.ctx, s.info) + s.responseMu.Lock() + resp := s.response + s.responseMu.Unlock() + if resp == nil { // We don't have a response stream to read from, try to get one. + newResponse, err := s.getter(s.ctx, s.info) if err != nil { return 0, err } // Successful GET; this is the network stream we'll read from. - s.response = response + s.setResponse(newResponse) + resp = newResponse } - n, err := s.response.Body.Read(p) // Read from the stream + n, err := resp.Body.Read(p) // Read from the stream (this will return non-nil err if forceRetry is called, from another goroutine, while it is running) // Injection mechanism for testing. if s.o.doInjectError && try == s.o.doInjectErrorRound { @@ -96,23 +135,49 @@ func (s *retryReader) Read(p []byte) (n int, err error) { } return n, err // Return the return to the caller } - s.Close() // Error, close stream - s.response = nil // Our stream is no longer good + s.Close() // Error, close stream + s.setResponse(nil) // Our stream is no longer good // Check the retry count and error code, and decide whether to retry. - if try >= s.o.MaxRetryRequests { - return n, err // All retries exhausted + retriesExhausted := try >= s.o.MaxRetryRequests + _, isNetError := err.(net.Error) + willRetry := (isNetError || s.wasRetryableEarlyClose(err)) && !retriesExhausted + + // Notify, for logging purposes, of any failures + if s.o.NotifyFailedRead != nil { + failureCount := try + 1 // because try is zero-based + s.o.NotifyFailedRead(failureCount, err, s.info.Offset, s.info.Count, willRetry) } - if _, ok := err.(net.Error); ok { + if willRetry { continue // Loop around and try to get and read from new stream. } - return n, err // Not retryable, just return + return n, err // Not retryable, or retries exhausted, so just return } } +// By default, we allow early Closing, from another concurrent goroutine, to be used to force a retry +// Is this safe, to close early from another goroutine? Early close ultimately ends up calling +// net.Conn.Close, and that is documented as "Any blocked Read or Write operations will be unblocked and return errors" +// which is exactly the behaviour we want. +// NOTE: that if caller has forced an early Close from a separate goroutine (separate from the Read) +// then there are two different types of error that may happen - either the one one we check for here, +// or a net.Error (due to closure of connection). Which one happens depends on timing. We only need this routine +// to check for one, since the other is a net.Error, which our main Read retry loop is already handing. +func (s *retryReader) wasRetryableEarlyClose(err error) bool { + if s.o.TreatEarlyCloseAsError { + return false // user wants all early closes to be errors, and so not retryable + } + // unfortunately, http.errReadOnClosedResBody is private, so the best we can do here is to check for its text + return strings.HasSuffix(err.Error(), ReadOnClosedBodyMessage) +} + +const ReadOnClosedBodyMessage = "read on closed response body" + func (s *retryReader) Close() error { + s.responseMu.Lock() + defer s.responseMu.Unlock() if s.response != nil && s.response.Body != nil { return s.response.Body.Close() } diff --git a/azbfs/zt_retry_reader_test.go b/azbfs/zt_retry_reader_test.go new file mode 100644 index 000000000..cc5b16321 --- /dev/null +++ b/azbfs/zt_retry_reader_test.go @@ -0,0 +1,330 @@ +package azbfs_test + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "github.com/Azure/azure-storage-azcopy/azbfs" + "io" + "net" + "net/http" + "time" + + chk "gopkg.in/check.v1" +) + +// Testings for RetryReader +// This reader return one byte through each Read call +type perByteReader struct { + RandomBytes []byte // Random generated bytes + + byteCount int // Bytes can be returned before EOF + currentByteIndex int // Bytes that have already been returned. + doInjectError bool + doInjectErrorByteIndex int + doInjectTimes int + injectedError error + + // sleepDuraion and closeChannel are only use in "forced cancellation" tests + sleepDuration time.Duration + closeChannel chan struct{} +} + +func newPerByteReader(byteCount int) *perByteReader { + perByteReader := perByteReader{ + byteCount: byteCount, + closeChannel: nil, + } + + perByteReader.RandomBytes = make([]byte, byteCount) + _, _ = rand.Read(perByteReader.RandomBytes) + + return &perByteReader +} + +func newSingleUsePerByteReader(contents []byte) *perByteReader { + perByteReader := perByteReader{ + byteCount: len(contents), + closeChannel: make(chan struct{}, 10), + } + + perByteReader.RandomBytes = contents + + return &perByteReader +} + +func (r *perByteReader) Read(b []byte) (n int, err error) { + if r.doInjectError && r.doInjectErrorByteIndex == r.currentByteIndex && r.doInjectTimes > 0 { + r.doInjectTimes-- + return 0, r.injectedError + } + + if r.currentByteIndex < r.byteCount { + n = copy(b, r.RandomBytes[r.currentByteIndex:r.currentByteIndex+1]) + r.currentByteIndex += n + + // simulate a delay, which may be successful or, if we're closed from another go-routine, may return an + // error + select { + case <-r.closeChannel: + return n, errors.New(azbfs.ReadOnClosedBodyMessage) + case <-time.After(r.sleepDuration): + return n, nil + } + } + + return 0, io.EOF +} + +func (r *perByteReader) Close() error { + if r.closeChannel != nil { + r.closeChannel <- struct{}{} + } + return nil +} + +// Test normal retry succeed, note initial response not provided. +// Tests both with and without notification of failures +func (r *aztestsSuite) TestRetryReaderReadWithRetry(c *chk.C) { + // Test twice, the second time using the optional "logging"/notification callback for failed tries + // We must test both with and without the callback, since be testing without + // we are testing that it is, indeed, optional to provide the callback + for _, logThisRun := range []bool{false, true} { + + // Extra setup for testing notification of failures (i.e. of unsuccessful tries) + failureMethodNumCalls := 0 + failureWillRetryCount := 0 + failureLastReportedFailureCount := -1 + var failureLastReportedError error = nil + failureMethod := func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + failureMethodNumCalls++ + if willRetry { + failureWillRetryCount++ + } + failureLastReportedFailureCount = failureCount + failureLastReportedError = lastError + } + + // Main test setup + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 0 + body.doInjectTimes = 1 + body.injectedError = &net.DNSError{IsTemporary: true} + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + httpGetterInfo := azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)} + initResponse, err := getter(context.Background(), httpGetterInfo) + c.Assert(err, chk.IsNil) + + rrOptions := azbfs.RetryReaderOptions{MaxRetryRequests: 1} + if logThisRun { + rrOptions.NotifyFailedRead = failureMethod + } + retryReader := azbfs.NewRetryReader(context.Background(), initResponse, httpGetterInfo, rrOptions, getter) + + // should fail and succeed through retry + can := make([]byte, 1) + n, err := retryReader.Read(can) + c.Assert(n, chk.Equals, 1) + c.Assert(err, chk.IsNil) + + // check "logging", if it was enabled + if logThisRun { + // We only expect one failed try in this test + // And the notification method is not called for successes + c.Assert(failureMethodNumCalls, chk.Equals, 1) // this is the number of calls we counted + c.Assert(failureWillRetryCount, chk.Equals, 1) // the sole failure was retried + c.Assert(failureLastReportedFailureCount, chk.Equals, 1) // this is the number of failures reported by the notification method + c.Assert(failureLastReportedError, chk.NotNil) + } + // should return EOF + n, err = retryReader.Read(can) + c.Assert(n, chk.Equals, 0) + c.Assert(err, chk.Equals, io.EOF) + } +} + +// Test normal retry fail as retry Count not enough. +func (r *aztestsSuite) TestRetryReaderReadNegativeNormalFail(c *chk.C) { + // Extra setup for testing notification of failures (i.e. of unsuccessful tries) + failureMethodNumCalls := 0 + failureWillRetryCount := 0 + failureLastReportedFailureCount := -1 + var failureLastReportedError error = nil + failureMethod := func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + failureMethodNumCalls++ + if willRetry { + failureWillRetryCount++ + } + failureLastReportedFailureCount = failureCount + failureLastReportedError = lastError + } + + // Main test setup + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 0 + body.doInjectTimes = 2 + body.injectedError = &net.DNSError{IsTemporary: true} + + startResponse := http.Response{} + startResponse.Body = body + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + rrOptions := azbfs.RetryReaderOptions{ + MaxRetryRequests: 1, + NotifyFailedRead: failureMethod} + retryReader := azbfs.NewRetryReader(context.Background(), &startResponse, azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)}, rrOptions, getter) + + // should fail + can := make([]byte, 1) + n, err := retryReader.Read(can) + c.Assert(n, chk.Equals, 0) + c.Assert(err, chk.Equals, body.injectedError) + + // Check that we recieved the right notification callbacks + // We only expect two failed tries in this test, but only one + // of the would have had willRetry = true + c.Assert(failureMethodNumCalls, chk.Equals, 2) // this is the number of calls we counted + c.Assert(failureWillRetryCount, chk.Equals, 1) // only the first failure was retried + c.Assert(failureLastReportedFailureCount, chk.Equals, 2) // this is the number of failures reported by the notification method + c.Assert(failureLastReportedError, chk.NotNil) +} + +// Test boundary case when Count equals to 0 and fail. +func (r *aztestsSuite) TestRetryReaderReadCount0(c *chk.C) { + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 1 + body.doInjectTimes = 1 + body.injectedError = &net.DNSError{IsTemporary: true} + + startResponse := http.Response{} + startResponse.Body = body + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + retryReader := azbfs.NewRetryReader(context.Background(), &startResponse, azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)}, azbfs.RetryReaderOptions{MaxRetryRequests: 1}, getter) + + // should consume the only byte + can := make([]byte, 1) + n, err := retryReader.Read(can) + c.Assert(n, chk.Equals, 1) + c.Assert(err, chk.IsNil) + + // should not read when Count=0, and should return EOF + n, err = retryReader.Read(can) + c.Assert(n, chk.Equals, 0) + c.Assert(err, chk.Equals, io.EOF) +} + +func (r *aztestsSuite) TestRetryReaderReadNegativeNonRetriableError(c *chk.C) { + byteCount := 1 + body := newPerByteReader(byteCount) + body.doInjectError = true + body.doInjectErrorByteIndex = 0 + body.doInjectTimes = 1 + body.injectedError = fmt.Errorf("not retriable error") + + startResponse := http.Response{} + startResponse.Body = body + + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + retryReader := azbfs.NewRetryReader(context.Background(), &startResponse, azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)}, azbfs.RetryReaderOptions{MaxRetryRequests: 2}, getter) + + dest := make([]byte, 1) + _, err := retryReader.Read(dest) + c.Assert(err, chk.Equals, body.injectedError) +} + +// Test the case where we programmatically force a retry to happen, via closing the body early from another goroutine +// Unlike the retries orchestrated elsewhere in this test file, which simulate network failures for the +// purposes of unit testing, here we are testing the cancellation mechanism that is exposed to +// consumers of the API, to allow programmatic forcing of retries (e.g. if the consumer deems +// the read to be taking too long, they may force a retry in the hope of better performance next time). +func (r *aztestsSuite) TestRetryReaderReadWithForcedRetry(c *chk.C) { + + for _, enableRetryOnEarlyClose := range []bool{false, true} { + + // use the notification callback, so we know that the retry really did happen + failureMethodNumCalls := 0 + failureMethod := func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + failureMethodNumCalls++ + } + + // Main test setup + byteCount := 10 // so multiple passes through read loop will be required + sleepDuration := 100 * time.Millisecond + randBytes := make([]byte, byteCount) + _, _ = rand.Read(randBytes) + getter := func(ctx context.Context, info azbfs.HTTPGetterInfo) (*http.Response, error) { + body := newSingleUsePerByteReader(randBytes) // make new one every time, since we force closes in this test, and its unusable after a close + body.sleepDuration = sleepDuration + r := http.Response{} + body.currentByteIndex = int(info.Offset) + r.Body = body + + return &r, nil + } + + httpGetterInfo := azbfs.HTTPGetterInfo{Offset: 0, Count: int64(byteCount)} + initResponse, err := getter(context.Background(), httpGetterInfo) + c.Assert(err, chk.IsNil) + + rrOptions := azbfs.RetryReaderOptions{MaxRetryRequests: 2, TreatEarlyCloseAsError: !enableRetryOnEarlyClose} + rrOptions.NotifyFailedRead = failureMethod + retryReader := azbfs.NewRetryReader(context.Background(), initResponse, httpGetterInfo, rrOptions, getter) + + // set up timed cancellation from separate goroutine + go func() { + time.Sleep(sleepDuration * 5) + retryReader.Close() + }() + + // do the read (should fail, due to forced cancellation, and succeed through retry) + output := make([]byte, byteCount) + n, err := io.ReadFull(retryReader, output) + if enableRetryOnEarlyClose { + c.Assert(n, chk.Equals, byteCount) + c.Assert(err, chk.IsNil) + c.Assert(output, chk.DeepEquals, randBytes) + c.Assert(failureMethodNumCalls, chk.Equals, 1) // assert that the cancellation did indeed happen + } else { + c.Assert(err, chk.NotNil) + } + } +} + +// End testings for RetryReader diff --git a/common/chunkedFileWriter.go b/common/chunkedFileWriter.go index 336a7ec68..6699e95ca 100644 --- a/common/chunkedFileWriter.go +++ b/common/chunkedFileWriter.go @@ -33,7 +33,7 @@ import ( type ChunkedFileWriter interface { // WaitToScheduleChunk blocks until enough RAM is available to handle the given chunk, then it - // "reserves" that amount of RAM in the CacheLimiter and returns. + // "reserves" that amount of RAM in the CacheLimiter and returns. WaitToScheduleChunk(ctx context.Context, id ChunkID, chunkSize int64) error // EnqueueChunk hands the given chunkContents over to the ChunkedFileWriter, to be written to disk. @@ -42,7 +42,7 @@ type ChunkedFileWriter interface { // While any error may be returned immediately, errors are more likely to be returned later, on either a subsequent // call to this routine or on the final return to Flush. // After the chunk is written to disk, its reserved memory byte allocation is automatically subtracted from the CacheLimiter. - EnqueueChunk(ctx context.Context, retryForcer func(), id ChunkID, chunkSize int64, chunkContents io.Reader) error + EnqueueChunk(ctx context.Context, id ChunkID, chunkSize int64, chunkContents io.Reader, retryable bool) error // Flush will block until all the chunks have been written to disk. err will be non-nil if and only in any chunk failed to write. // Flush must be called exactly once, after all chunks have been enqueued with EnqueueChunk. @@ -123,7 +123,7 @@ const maxDesirableActiveChunks = 20 // TODO: can we find a sensible way to remov // at the time of scheduling the chunk (which is when this routine should be called). // Is here, as method of this struct, for symmetry with the point where we remove it's count // from the cache limiter, which is also in this struct. -func (w *chunkedFileWriter) WaitToScheduleChunk(ctx context.Context, id ChunkID, chunkSize int64) error{ +func (w *chunkedFileWriter) WaitToScheduleChunk(ctx context.Context, id ChunkID, chunkSize int64) error { w.chunkLogger.LogChunkStatus(id, EWaitReason.RAMToSchedule()) err := w.cacheLimiter.WaitUntilAddBytes(ctx, chunkSize, w.shouldUseRelaxedRamThreshold) if err == nil { @@ -133,9 +133,18 @@ func (w *chunkedFileWriter) WaitToScheduleChunk(ctx context.Context, id ChunkID, } // Threadsafe method to enqueue a new chunk for processing -func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, retryForcer func(), id ChunkID, chunkSize int64, chunkContents io.Reader) error { +func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, id ChunkID, chunkSize int64, chunkContents io.Reader, retryable bool) error { + readDone := make(chan struct{}) - w.setupProgressMonitoring(readDone, id, chunkSize, retryForcer) + if retryable { + // if retryable == true, that tells us that closing the reader + // is a safe way to force this particular reader to retry. + // (Typically this means it forces the reader to make one iteration around its internal retry loop. + // Going around that loop is hidden to the normal Read code (unless it exceeds the retry count threshold)) + closer := chunkContents.(io.Closer).Close // do the type assertion now, so get panic if it's not compatible. If we left it to the last minute, then the type would only be verified on the rare occasions when retries are required + retryForcer := func() { _ = closer() } + w.setupProgressMonitoring(readDone, id, chunkSize, retryForcer) + } // read into a buffer buffer := w.slicePool.RentSlice(uint32(chunkSize)) @@ -269,7 +278,6 @@ func (w *chunkedFileWriter) shouldUseRelaxedRamThreshold() bool { return atomic.LoadInt32(&w.activeChunkCount) <= maxDesirableActiveChunks } - // Are we currently in a memory-constrained situation? func (w *chunkedFileWriter) haveMemoryPressure(chunkSize int64) bool { didAdd := w.cacheLimiter.TryAddBytes(chunkSize, w.shouldUseRelaxedRamThreshold()) @@ -285,7 +293,7 @@ func (w *chunkedFileWriter) haveMemoryPressure(chunkSize int64) bool { // By retrying the slow chunk, we usually get a fast read. func (w *chunkedFileWriter) setupProgressMonitoring(readDone chan struct{}, id ChunkID, chunkSize int64, retryForcer func()) { if retryForcer == nil { - panic("retryForcer is nil. This probably means that the request pipeline is not producing cancelable requests. I.e. it is not producing response bodies that implement RequestCanceller") + panic("retryForcer is nil") } start := time.Now() initialReceivedCount := atomic.LoadInt32(&w.totalReceivedChunkCount) diff --git a/common/logger.go b/common/logger.go index be4751072..6df504721 100644 --- a/common/logger.go +++ b/common/logger.go @@ -21,7 +21,9 @@ package common import ( + "fmt" "log" + "net/url" "os" "runtime" @@ -165,3 +167,31 @@ func (jl jobLogger) Panic(err error) { jl.appLogger.Panic(err) // We panic here that it logs and the app terminates // We should never reach this line of code! } + +const TryEquals string = "Try=" // TODO: refactor so that this can be used by the retry policies too? So that when you search the logs for Try= you are guaranteed to find both types of retry (i.e. request send retries, and body read retries) + +func NewReadLogFunc(logger ILogger, fullUrl *url.URL) func(int, error, int64, int64, bool) { + redactedUrl := URLStringExtension(fullUrl.String()).RedactSigQueryParamForLogging() + + return func(failureCount int, err error, offset int64, count int64, willRetry bool) { + retryMessage := "Will retry" + if !willRetry { + retryMessage = "Will NOT retry" + } + logger.Log(pipeline.LogInfo, fmt.Sprintf( + "Error reading body of reply. Next try (if any) will be %s%d. %s. Error: %s. Offset: %d Count: %d URL: %s", + TryEquals, // so that retry wording for body-read retries is similar to that for URL-hitting retries + + // We log the number of the NEXT try, not the failure just done, so that users searching the log for "Try=2" + // will find ALL retries, both the request send retries (which are logged as try 2 when they are made) and + // body read retries (for which only the failure is logged - so if we did the actual failure number, there would be + // not Try=2 in the logs if the retries work). + failureCount+1, + + retryMessage, + err, + offset, + count, + redactedUrl)) + } +} diff --git a/ste/downloader-azureFiles.go b/ste/downloader-azureFiles.go index 3580b59ab..19e434de3 100644 --- a/ste/downloader-azureFiles.go +++ b/ste/downloader-azureFiles.go @@ -34,7 +34,7 @@ func newAzureFilesDownloader() downloader { return &azureFilesDownloader{} } -// Returns a chunk-func for file downloads +// GenerateDownloadFunc returns a chunk-func for file downloads func (bd *azureFilesDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPipeline pipeline.Pipeline, destWriter common.ChunkedFileWriter, id common.ChunkID, length int64, pacer *pacer) chunkFunc { return createDownloadChunkFunc(jptm, id, func() { @@ -56,14 +56,12 @@ func (bd *azureFilesDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, s // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) - retryReader := get.Body(azfile.RetryReaderOptions{MaxRetryRequests: MaxRetryPerDownloadBody}) + retryReader := get.Body(azfile.RetryReaderOptions{ + MaxRetryRequests: MaxRetryPerDownloadBody, + NotifyFailedRead: common.NewReadLogFunc(jptm, u), + }) defer retryReader.Close() - retryForcer := func() { - // TODO: implement this, or implement GetBodyWithForceableRetry above - // for now, this "retry forcer" does nothing - //fmt.Printf("\nForcing retry\n") - } - err = destWriter.EnqueueChunk(jptm.Context(), retryForcer, id, length, newLiteResponseBodyPacer(retryReader, pacer)) + err = destWriter.EnqueueChunk(jptm.Context(), id, length, newLiteResponseBodyPacer(retryReader, pacer), true) if err != nil { jptm.FailActiveDownload("Enqueuing chunk", err) return diff --git a/ste/downloader-blob.go b/ste/downloader-blob.go index 6e860b71f..2ab9c45af 100644 --- a/ste/downloader-blob.go +++ b/ste/downloader-blob.go @@ -54,13 +54,12 @@ func (bd *blobDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPipe // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) - //TODO: retryReader, retryForcer := get.BodyWithForceableRetry(azblob.RetryReaderOptions{MaxRetryRequests: MaxRetryPerDownloadBody}) - retryReader := get.Body(azblob.RetryReaderOptions{MaxRetryRequests: destWriter.MaxRetryPerDownloadBody()}) - retryForcer := func() {} - // TODO: replace the above with real retry forcer - + retryReader := get.Body(azblob.RetryReaderOptions{ + MaxRetryRequests: destWriter.MaxRetryPerDownloadBody(), + NotifyFailedRead: common.NewReadLogFunc(jptm, u), + }) defer retryReader.Close() - err = destWriter.EnqueueChunk(jptm.Context(), retryForcer, id, length, newLiteResponseBodyPacer(retryReader, pacer)) + err = destWriter.EnqueueChunk(jptm.Context(), id, length, newLiteResponseBodyPacer(retryReader, pacer), true) if err != nil { jptm.FailActiveDownload("Enqueuing chunk", err) return diff --git a/ste/downloader-blobFS.go b/ste/downloader-blobFS.go index e49883be9..a99e91ac7 100644 --- a/ste/downloader-blobFS.go +++ b/ste/downloader-blobFS.go @@ -55,14 +55,12 @@ func (bd *blobFSDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPi // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) - retryReader := get.Body(azbfs.RetryReaderOptions{MaxRetryRequests: MaxRetryPerDownloadBody}) + retryReader := get.Body(azbfs.RetryReaderOptions{ + MaxRetryRequests: MaxRetryPerDownloadBody, + NotifyFailedRead: common.NewReadLogFunc(jptm, u), + }) defer retryReader.Close() - retryForcer := func() { - // TODO: implement this, or implement GetBodyWithForceableRetry above - // for now, this "retry forcer" does nothing - //fmt.Printf("\nForcing retry\n") - } - err = destWriter.EnqueueChunk(jptm.Context(), retryForcer, id, length, newLiteResponseBodyPacer(retryReader, pacer)) + err = destWriter.EnqueueChunk(jptm.Context(), id, length, newLiteResponseBodyPacer(retryReader, pacer), true) if err != nil { jptm.FailActiveDownload("Enqueuing chunk", err) return From 61b44556fba545b0ab27843faf60ae95a5c568eb Mon Sep 17 00:00:00 2001 From: John Rusk Date: Thu, 7 Feb 2019 18:35:08 +1300 Subject: [PATCH 03/64] Remove unwanted and erroneous return statement from mod-date-setter error path (#200) --- ste/xfer-remoteToLocal.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ste/xfer-remoteToLocal.go b/ste/xfer-remoteToLocal.go index fd769deb3..684eff11a 100644 --- a/ste/xfer-remoteToLocal.go +++ b/ste/xfer-remoteToLocal.go @@ -199,9 +199,8 @@ func epilogueWithCleanupDownload(jptm IJobPartTransferMgr, activeDstFile *os.Fil err := os.Chtimes(jptm.Info().Destination, lastModifiedTime, lastModifiedTime) if err != nil { jptm.LogError(info.Destination, "Changing Modified Time ", err) - return - } - if jptm.ShouldLog(pipeline.LogInfo) { + // do NOT return, since final status and cleanup logging still to come + } else { jptm.Log(pipeline.LogInfo, fmt.Sprintf(" Preserved Modified Time for %s", info.Destination)) } } From fd4358c7145c78d23189369c5b97a567d71a0bde Mon Sep 17 00:00:00 2001 From: John Rusk Date: Thu, 7 Feb 2019 18:40:58 +1300 Subject: [PATCH 04/64] Improve logging of HTTP errors (#201) * Append extra newline to serious errors to separate them from unrlated stuff that may follow them * Include X-ms-request-id in transfer failed log messages To alllow correlation of the failure with the request that triggered it --- ste/ErrorExt.go | 18 ++++++++++++++++++ ste/mgr-JobPartTransferMgr.go | 7 +++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ste/ErrorExt.go b/ste/ErrorExt.go index b3fcc567d..d84e7edfe 100644 --- a/ste/ErrorExt.go +++ b/ste/ErrorExt.go @@ -4,12 +4,14 @@ import ( "github.com/Azure/azure-storage-azcopy/azbfs" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/azure-storage-file-go/azfile" + "net/http" ) type ErrorEx struct { error } +// TODO: consider rolling MSRequestID into this, so that all places that use this can pick up, and log, the request ID too func (errex ErrorEx) ErrorCodeAndString() (int, string) { switch e := interface{}(errex.error).(type) { case azblob.StorageError: @@ -22,3 +24,19 @@ func (errex ErrorEx) ErrorCodeAndString() (int, string) { return 0, errex.Error() } } + +type hasResponse interface { + Response() *http.Response +} + +// MSRequestID gets the request ID guid associated with the failed request. +// Returns "" if there isn't one (either no request, or there is a request but it doesn't have the header) +func (errex ErrorEx) MSRequestID() string { + if respErr, ok := errex.error.(hasResponse); ok { + r := respErr.Response() + if r != nil { + return r.Header.Get("X-Ms-Request-Id") + } + } + return "" +} diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index d7ed11301..9993dd64c 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -332,7 +332,9 @@ func (jptm *jobPartTransferMgr) failActiveTransfer(typ transferErrorCode, descri if !jptm.WasCanceled() { jptm.Cancel() status, msg := ErrorEx{err}.ErrorCodeAndString() - jptm.logTransferError(typ, jptm.Info().Source, jptm.Info().Destination, msg+" when "+descriptionOfWhereErrorOccurred, status) + requestID := ErrorEx{err}.MSRequestID() + fullMsg := fmt.Sprintf("%s. When %s. X-Ms-Request-Id: %s\n", msg, descriptionOfWhereErrorOccurred, requestID) // trailing \n to separate it better from any later, unrelated, log lines + jptm.logTransferError(typ, jptm.Info().Source, jptm.Info().Destination, fullMsg, status) jptm.SetStatus(failureStatus) jptm.SetErrorCode(int32(status)) // TODO: what are the rules about when this needs to be set, and doesn't need to be (e.g. for earlier failures)? // If the status code was 403, it means there was an authentication error and we exit. @@ -402,8 +404,9 @@ func (jptm *jobPartTransferMgr) LogS2SCopyError(source, destination, errorMsg st func (jptm *jobPartTransferMgr) LogError(resource, context string, err error) { status, msg := ErrorEx{err}.ErrorCodeAndString() + MSRequestID := ErrorEx{err}.MSRequestID() jptm.Log(pipeline.LogError, - fmt.Sprintf("%s: %d: %s-%s", common.URLStringExtension(resource).RedactSigQueryParamForLogging(), status, context, msg)) + fmt.Sprintf("%s: %d: %s-%s. X-Ms-Request-Id:%s\n", common.URLStringExtension(resource).RedactSigQueryParamForLogging(), status, context, msg, MSRequestID)) } func (jptm *jobPartTransferMgr) LogTransferStart(source, destination, description string) { From 07fcae11f6966873b1a02383e55f7430c6592282 Mon Sep 17 00:00:00 2001 From: John Rusk Date: Fri, 8 Feb 2019 13:18:57 +1300 Subject: [PATCH 05/64] Avoid needless panic after cancellation of block blob upload (#202) --- ste/uploader-blockBlob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ste/uploader-blockBlob.go b/ste/uploader-blockBlob.go index e093b5fc6..91c2f89cd 100644 --- a/ste/uploader-blockBlob.go +++ b/ste/uploader-blockBlob.go @@ -172,7 +172,7 @@ func (u *blockBlobUploader) Epilogue() { blockIds := u.blockIds u.mu.Unlock() shouldPutBlockList := getPutListNeed(&u.putListIndicator) - if shouldPutBlockList == putListNeedUnknown { + if shouldPutBlockList == putListNeedUnknown && !jptm.WasCanceled() { panic("'put list' need flag was never set") } From 5b1ccd2944f916e12317576b4be33a0d2967d2f3 Mon Sep 17 00:00:00 2001 From: John Rusk Date: Fri, 8 Feb 2019 22:40:09 +1300 Subject: [PATCH 06/64] Feature/md5 save and check (#203) * Make prefetch have an error code Since we need to know whether prefetch happened to compute hashes. * PUT whole-of-file MD5s blobFS is not covered by this commit. But Block, Page and Append blobs are, and so are Azure Files * Update comment * Check MD5s when downloading * Refactor schedulding of upload chunks for testability So that the main scheduling routine can be tested in isolation of all the precursor setup stuff * Add missing captures of ContentMD5 during enumeration * Remove resolved TODO * Rename comparison type to md5Comparer * Add command-line options to control the strictness of MD5 validation * Trigger CI * Fix missing hashValidationOption in remove command --- cmd/copy.go | 24 ++++++++ cmd/copyDownloadBlobEnumerator.go | 13 +++- cmd/copyDownloadFileEnumerator.go | 3 +- cmd/remove.go | 3 +- cmd/sync.go | 34 +++++++---- cmd/syncDownloadEnumerator.go | 5 ++ common/chunkedFileWriter.go | 40 ++++++++----- common/emptyChunkReader.go | 9 ++- common/fe-ste-models.go | 46 ++++++++++++++ common/rpc-models.go | 17 +++--- common/singleChunkReader.go | 38 ++++++------ ste/JobPartPlan.go | 5 +- ste/JobPartPlanFileName.go | 5 +- ste/md5Comparer.go | 99 +++++++++++++++++++++++++++++++ ste/mgr-JobPartMgr.go | 12 ++-- ste/mgr-JobPartTransferMgr.go | 18 +++++- ste/uploader-appendBlob.go | 20 +++++++ ste/uploader-azureFiles.go | 44 ++++++++++---- ste/uploader-blobFS.go | 31 ++++++---- ste/uploader-blockBlob.go | 38 ++++++++++-- ste/uploader-pageBlob.go | 31 ++++++++-- ste/uploader.go | 26 ++++++++ ste/xfer-localToRemote.go | 75 +++++++++++++++++------ ste/xfer-remoteToLocal.go | 36 ++++++----- 24 files changed, 531 insertions(+), 141 deletions(-) create mode 100644 ste/md5Comparer.go diff --git a/cmd/copy.go b/cmd/copy.go index 5ecb28b5c..d5e342e22 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -84,6 +84,7 @@ type rawCopyCmdArgs struct { contentEncoding string noGuessMimeType bool preserveLastModifiedTime bool + md5ValidationOption string // defines the type of the blob at the destination in case of upload / account to account copy blobType string blockBlobTier string @@ -225,6 +226,12 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { cooked.contentEncoding = raw.contentEncoding cooked.noGuessMimeType = raw.noGuessMimeType cooked.preserveLastModifiedTime = raw.preserveLastModifiedTime + + err = cooked.md5ValidationOption.Parse(raw.md5ValidationOption) + if err != nil { + return cooked, err + } + cooked.background = raw.background cooked.acl = raw.acl cooked.cancelFromStdin = raw.cancelFromStdin @@ -289,6 +296,9 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { return cooked, fmt.Errorf("content-type, content-encoding or metadata is set while copying from sevice to service") } } + if err = validateMd5Option(cooked.md5ValidationOption, cooked.fromTo); err != nil { + return cooked, err + } // If the user has provided some input with excludeBlobType flag, parse the input. if len(raw.excludeBlobType) > 0 { @@ -307,6 +317,15 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { return cooked, nil } +func validateMd5Option(option common.HashValidationOption, fromTo common.FromTo) error { + hasMd5Validation := option != common.DefaultHashValidationOption + isDownload := fromTo.To() == common.ELocation.Local() + if hasMd5Validation && !isDownload { + return fmt.Errorf("md5-validation is set but the job is not a download") + } + return nil +} + // represents the processed copy command input from the user type cookedCopyCmdArgs struct { // from arguments @@ -337,6 +356,7 @@ type cookedCopyCmdArgs struct { contentEncoding string noGuessMimeType bool preserveLastModifiedTime bool + md5ValidationOption common.HashValidationOption background bool output common.OutputFormat acl string @@ -509,6 +529,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { Metadata: cca.metadata, NoGuessMimeType: cca.noGuessMimeType, PreserveLastModifiedTime: cca.preserveLastModifiedTime, + MD5ValidationOption: cca.md5ValidationOption, }, // source sas is stripped from the source given by the user and it will not be stored in the part plan file. SourceSAS: cca.sourceSAS, @@ -906,6 +927,9 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.contentEncoding, "content-encoding", "", "upload to Azure Storage using this content encoding.") cpCmd.PersistentFlags().BoolVar(&raw.noGuessMimeType, "no-guess-mime-type", false, "prevents AzCopy from detecting the content-type based on the extension/content of the file.") cpCmd.PersistentFlags().BoolVar(&raw.preserveLastModifiedTime, "preserve-last-modified-time", false, "only available when destination is file system.") + cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading.") + // TODO: should the previous line list the allowable values? + cpCmd.PersistentFlags().BoolVar(&raw.cancelFromStdin, "cancel-from-stdin", false, "true if user wants to cancel the process by passing 'cancel' "+ "to the standard input. This is mostly used when the application is spawned by another process.") cpCmd.PersistentFlags().BoolVar(&raw.background, "background-op", false, "true if user has to perform the operations as a background operation.") diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index c253f7a4d..5824f245f 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -72,6 +72,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: blobLocalPath, LastModifiedTime: blobProperties.LastModified(), SourceSize: blobProperties.ContentLength(), + ContentMD5: blobProperties.ContentMD5(), }, cca) // only one transfer for this Job, dispatch the JobPart err := e.dispatchFinalPart(cca) @@ -150,7 +151,9 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobPath)).String(), Destination: util.generateLocalPath(cca.destination, blobRelativePath), LastModifiedTime: blobProperties.LastModified(), - SourceSize: blobProperties.ContentLength()}, cca) + SourceSize: blobProperties.ContentLength(), + ContentMD5: blobProperties.ContentMD5(), + }, cca) continue } if !cca.recursive { @@ -201,7 +204,9 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name)).String(), Destination: util.generateLocalPath(cca.destination, blobRelativePath), LastModifiedTime: blobInfo.Properties.LastModified, - SourceSize: *blobInfo.Properties.ContentLength}, cca) + SourceSize: *blobInfo.Properties.ContentLength, + ContentMD5: blobInfo.Properties.ContentMD5, + }, cca) } marker = listBlob.NextMarker } @@ -290,7 +295,9 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name)).String(), Destination: util.generateLocalPath(cca.destination, blobRelativePath), LastModifiedTime: blobInfo.Properties.LastModified, - SourceSize: *blobInfo.Properties.ContentLength}, cca) + SourceSize: *blobInfo.Properties.ContentLength, + ContentMD5: blobInfo.Properties.ContentMD5, + }, cca) } marker = listBlob.NextMarker } diff --git a/cmd/copyDownloadFileEnumerator.go b/cmd/copyDownloadFileEnumerator.go index b6ee37fc5..9357fbf9e 100644 --- a/cmd/copyDownloadFileEnumerator.go +++ b/cmd/copyDownloadFileEnumerator.go @@ -155,7 +155,8 @@ func (e *copyDownloadFileEnumerator) addDownloadFileTransfer(srcURL url.URL, des Source: gCopyUtil.stripSASFromFileShareUrl(srcURL).String(), Destination: destPath, LastModifiedTime: properties.LastModified(), - SourceSize: properties.ContentLength()}, + SourceSize: properties.ContentLength(), + ContentMD5: properties.ContentMD5()}, cca) } diff --git a/cmd/remove.go b/cmd/remove.go index 672d2e883..53278e6ab 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -51,9 +51,10 @@ func init() { } else { return fmt.Errorf("invalid source type %s pased to delete. azcopy support removing blobs and files only", srcLocationType.String()) } - // Since remove uses the copy command arguments cook, set the blobType to None + // Since remove uses the copy command arguments cook, set the blobType to None and validation option // else parsing the arguments will fail. raw.blobType = common.EBlobType.None().String() + raw.md5ValidationOption = common.DefaultHashValidationOption.String() return nil }, Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/sync.go b/cmd/sync.go index 51f845be8..72649f279 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -46,12 +46,13 @@ type rawSyncCmdArgs struct { dst string recursive bool // options from flags - blockSize uint32 - logVerbosity string - include string - exclude string - followSymlinks bool - output string + blockSize uint32 + logVerbosity string + include string + exclude string + followSymlinks bool + output string + md5ValidationOption string // this flag predefines the user-agreement to delete the files in case sync found some files at destination // which doesn't exists at source. With this flag turned on, user will not be asked for permission before // deleting the flag. @@ -84,6 +85,14 @@ func (raw rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { return cooked, err } + err = cooked.md5ValidationOption.Parse(raw.md5ValidationOption) + if err != nil { + return cooked, err + } + if err = validateMd5Option(cooked.md5ValidationOption, cooked.fromTo); err != nil { + return cooked, err + } + // initialize the include map which contains the list of files to be included // parse the string passed in include flag // more than one file are expected to be separated by ';' @@ -132,11 +141,12 @@ type cookedSyncCmdArgs struct { recursive bool followSymlinks bool // options from flags - include map[string]int - exclude map[string]int - blockSize uint32 - logVerbosity common.LogLevel - output common.OutputFormat + include map[string]int + exclude map[string]int + blockSize uint32 + logVerbosity common.LogLevel + output common.OutputFormat + md5ValidationOption common.HashValidationOption // commandString hold the user given command which is logged to the Job log file commandString string @@ -490,6 +500,8 @@ func init() { syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "WARNING", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") syncCmd.PersistentFlags().BoolVar(&raw.force, "force", false, "defines user's decision to delete extra files at the destination that are not present at the source. "+ "If false, user will be prompted with a question while scheduling files/blobs for deletion.") + syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading.") + // TODO: should the previous line list the allowable values? // TODO sync does not support any BlobAttributes, this functionality should be added } diff --git a/cmd/syncDownloadEnumerator.go b/cmd/syncDownloadEnumerator.go index fafe2280c..a4d62d6c3 100644 --- a/cmd/syncDownloadEnumerator.go +++ b/cmd/syncDownloadEnumerator.go @@ -215,6 +215,7 @@ func (e *syncDownloadEnumerator) listSourceAndCompare(cca *cookedSyncCmdArgs, p Destination: blobLocalPath, SourceSize: *blobInfo.Properties.ContentLength, LastModifiedTime: blobInfo.Properties.LastModified, + ContentMD5: blobInfo.Properties.ContentMD5, }, cca) delete(e.SourceFiles, blobLocalPath) @@ -295,6 +296,7 @@ func (e *syncDownloadEnumerator) listTheDestinationIfRequired(cca *cookedSyncCmd Destination: cca.source, SourceSize: bProperties.ContentLength(), LastModifiedTime: bProperties.LastModified(), + ContentMD5: bProperties.ContentMD5(), }, cca) } @@ -417,6 +419,9 @@ func (e *syncDownloadEnumerator) enumerate(cca *cookedSyncCmdArgs) error { // Set the preserve-last-modified-time to true in CopyJobRequest e.CopyJobRequest.BlobAttributes.PreserveLastModifiedTime = true + // set MD5 validation behaviour + e.CopyJobRequest.BlobAttributes.MD5ValidationOption = cca.md5ValidationOption + // Copying the JobId of sync job to individual deleteJobRequest. e.DeleteJobRequest.JobID = e.JobID // FromTo of DeleteJobRequest will be BlobTrash. diff --git a/common/chunkedFileWriter.go b/common/chunkedFileWriter.go index 6699e95ca..869dd808b 100644 --- a/common/chunkedFileWriter.go +++ b/common/chunkedFileWriter.go @@ -22,7 +22,9 @@ package common import ( "context" + "crypto/md5" "errors" + "hash" "io" "math" "sync/atomic" @@ -46,7 +48,7 @@ type ChunkedFileWriter interface { // Flush will block until all the chunks have been written to disk. err will be non-nil if and only in any chunk failed to write. // Flush must be called exactly once, after all chunks have been enqueued with EnqueueChunk. - Flush(ctx context.Context) (md5Hash string, err error) + Flush(ctx context.Context) (md5HashOfFileAsWritten []byte, err error) // MaxRetryPerDownloadBody returns the maximum number of retries that will be done for the download of a single chunk body MaxRetryPerDownloadBody() int @@ -79,7 +81,7 @@ type chunkedFileWriter struct { creationTime time.Time // used for completion - successMd5 chan string // TODO: use this when we do MD5s + successMd5 chan []byte failureError chan error // controls body-read retries. Public so value can be shared with retryReader @@ -102,7 +104,7 @@ func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheL slicePool: slicePool, cacheLimiter: cacheLimiter, chunkLogger: chunkLogger, - successMd5: make(chan string), + successMd5: make(chan []byte), failureError: make(chan error, 1), newUnorderedChunks: make(chan fileChunk, chanBufferSize), creationTime: time.Now(), @@ -169,8 +171,8 @@ func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, id ChunkID, chunkS } } -// Waits until all chunks have been flush to disk, then returns -func (w *chunkedFileWriter) Flush(ctx context.Context) (string, error) { +// Flush waits until all chunks have been flush to disk, then returns the MD5 has of the file's bytes-as-we-saved-them +func (w *chunkedFileWriter) Flush(ctx context.Context) ([]byte, error) { // let worker know that no more will be coming close(w.newUnorderedChunks) @@ -178,13 +180,13 @@ func (w *chunkedFileWriter) Flush(ctx context.Context) (string, error) { select { case err := <-w.failureError: if err != nil { - return "", err + return nil, err } - return "", ChunkWriterAlreadyFailed // channel returned nil because it was closed and empty + return nil, ChunkWriterAlreadyFailed // channel returned nil because it was closed and empty case <-ctx.Done(): - return "", ctx.Err() - case hashAsAtCompletion := <-w.successMd5: - return hashAsAtCompletion, nil + return nil, ctx.Err() + case md5AtCompletion := <-w.successMd5: + return md5AtCompletion, nil } } @@ -200,6 +202,7 @@ func (w *chunkedFileWriter) MaxRetryPerDownloadBody() int { func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { nextOffsetToSave := int64(0) unsavedChunksByFileOffset := make(map[int64]fileChunk) + md5Hasher := md5.New() for { var newChunk fileChunk @@ -211,8 +214,8 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { if !channelIsOpen { // If channel is closed, we know that flush as been called and we have read everything // So we are finished - // TODO: add returning of MD5 hash in the next line - w.successMd5 <- "" // everything is done. We know there was no error, because if there was an error we would have returned before now + // We know there was no error, because if there was an error we would have returned before now + w.successMd5 <- md5Hasher.Sum(nil) return } case <-ctx.Done(): @@ -226,7 +229,7 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { w.chunkLogger.LogChunkStatus(newChunk.id, EWaitReason.PriorChunk()) // may have to wait on prior chunks to arrive // Process all chunks that we can - err := w.saveAvailableChunks(unsavedChunksByFileOffset, &nextOffsetToSave) + err := w.sequentiallyProcessAvailableChunks(unsavedChunksByFileOffset, &nextOffsetToSave, md5Hasher) if err != nil { w.failureError <- err close(w.failureError) // must close because many goroutines may be calling the public methods, and all need to be able to tell there's been an error, even tho only one will get the actual error @@ -235,16 +238,21 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { } } -// Saves available chunks that are sequential from nextOffsetToSave. Stops and returns as soon as it hits +// Hashes and saves available chunks that are sequential from nextOffsetToSave. Stops and returns as soon as it hits // a gap (i.e. the position of a chunk that hasn't arrived yet) -func (w *chunkedFileWriter) saveAvailableChunks(unsavedChunksByFileOffset map[int64]fileChunk, nextOffsetToSave *int64) error { +func (w *chunkedFileWriter) sequentiallyProcessAvailableChunks(unsavedChunksByFileOffset map[int64]fileChunk, nextOffsetToSave *int64, md5Hasher hash.Hash) error { for { + // Look for next chunk in sequence nextChunkInSequence, exists := unsavedChunksByFileOffset[*nextOffsetToSave] if !exists { return nil //its not there yet. That's OK. } - *nextOffsetToSave += int64(len(nextChunkInSequence.data)) + *nextOffsetToSave += int64(len(nextChunkInSequence.data)) // update immediately so we won't forget! + // Add it to the hash (must do so sequentially for MD5) + md5Hasher.Write(nextChunkInSequence.data) + + // Save it err := w.saveOneChunk(nextChunkInSequence) if err != nil { return err diff --git a/common/emptyChunkReader.go b/common/emptyChunkReader.go index 6eba3ff13..455ff365c 100644 --- a/common/emptyChunkReader.go +++ b/common/emptyChunkReader.go @@ -22,6 +22,7 @@ package common import ( "errors" + "hash" "io" ) @@ -29,8 +30,8 @@ import ( type emptyChunkReader struct { } -func (cr *emptyChunkReader) TryBlockingPrefetch(fileReader io.ReaderAt) bool { - return true +func (cr *emptyChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { + return nil } func (cr *emptyChunkReader) Seek(offset int64, whence int) (int64, error) { @@ -59,3 +60,7 @@ func (cr *emptyChunkReader) HasPrefetchedEntirelyZeros() bool { func (cr *emptyChunkReader) Length() int64 { return 0 } + +func (cr *emptyChunkReader) WriteBufferTo(h hash.Hash) { + return // no content to write +} diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 0e99575c5..2bff1d6b2 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -524,6 +524,52 @@ func (ct *CredentialType) Parse(s string) error { return err } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +var EHashValidationOption = HashValidationOption(0) + +var DefaultHashValidationOption = EHashValidationOption.FailIfDifferent() + +type HashValidationOption uint8 + +// LogOnly is the least strict option +func (HashValidationOption) LogOnly() HashValidationOption { return HashValidationOption(0) } + +// FailIfDifferent says fail if hashes different, but NOT fail if saved hash is +// totally missing. This is a balance of convenience (for cases where no hash is saved) vs strictness +// (to validate strictly when one is present) +func (HashValidationOption) FailIfDifferent() HashValidationOption { return HashValidationOption(1) } + +// FailIfDifferentOrMissing is the strictest option, and useful for testing or validation in cases when +// we _know_ there should be a hash +func (HashValidationOption) FailIfDifferentOrMissing() HashValidationOption { + return HashValidationOption(2) +} + +func (hvo HashValidationOption) String() string { + return enum.StringInt(hvo, reflect.TypeOf(hvo)) +} + +func (hvo *HashValidationOption) Parse(s string) error { + val, err := enum.ParseInt(reflect.TypeOf(hvo), s, true, true) + if err == nil { + *hvo = val.(HashValidationOption) + } + return err +} + +func (hvo HashValidationOption) MarshalJSON() ([]byte, error) { + return json.Marshal(hvo.String()) +} + +func (hvo *HashValidationOption) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + return hvo.Parse(s) +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const ( DefaultBlockBlobBlockSize = 8 * 1024 * 1024 diff --git a/common/rpc-models.go b/common/rpc-models.go index 4fed36879..931cc498d 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -108,14 +108,15 @@ type ListRequest struct { // This struct represents the optional attribute for blob request header type BlobTransferAttributes struct { - BlobType BlobType // The type of a blob - BlockBlob, PageBlob, AppendBlob - ContentType string //The content type specified for the blob. - ContentEncoding string //Specifies which content encodings have been applied to the blob. - BlockBlobTier BlockBlobTier // Specifies the tier to set on the block blobs. - PageBlobTier PageBlobTier // Specifies the tier to set on the page blobs. - Metadata string //User-defined Name-value pairs associated with the blob - NoGuessMimeType bool // represents user decision to interpret the content-encoding from source file - PreserveLastModifiedTime bool // when downloading, tell engine to set file's timestamp to timestamp of blob + BlobType BlobType // The type of a blob - BlockBlob, PageBlob, AppendBlob + ContentType string //The content type specified for the blob. + ContentEncoding string //Specifies which content encodings have been applied to the blob. + BlockBlobTier BlockBlobTier // Specifies the tier to set on the block blobs. + PageBlobTier PageBlobTier // Specifies the tier to set on the page blobs. + Metadata string //User-defined Name-value pairs associated with the blob + NoGuessMimeType bool // represents user decision to interpret the content-encoding from source file + PreserveLastModifiedTime bool // when downloading, tell engine to set file's timestamp to timestamp of blob + MD5ValidationOption HashValidationOption // when downloading, how strictly should we validate MD5 hashes? BlockSizeInBytes uint32 } diff --git a/common/singleChunkReader.go b/common/singleChunkReader.go index eb42f0b9b..ac0d45a6a 100644 --- a/common/singleChunkReader.go +++ b/common/singleChunkReader.go @@ -23,6 +23,7 @@ package common import ( "context" "errors" + "hash" "io" ) @@ -42,11 +43,8 @@ type SingleChunkReader interface { // Closer is needed to clean up resources io.Closer - // TryBlockingPrefetch tries to read the full contents of the chunk into RAM. Returns true if succeeded for false if failed, - // although callers do not have to check the return value (and there is no error object returned). Why? - // Because its OK to keep using this object even if the prefetch fails, since in that case - // any subsequent Read will just retry the same read as we do here, and if it fails at that time then Read will return an error. - TryBlockingPrefetch(fileReader io.ReaderAt) bool + // BlockingPrefetch tries to read the full contents of the chunk into RAM. + BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error // CaptureLeadingBytes is used to grab enough of the initial bytes to do MIME-type detection. Expected to be called only // on the first chunk in each file (since there's no point in calling it on others) @@ -61,6 +59,10 @@ type SingleChunkReader interface { // In the rare edge case where this returns false due to the prefetch having failed (rather than the contents being non-zero), // we'll just treat it as a non-zero chunk. That's simpler (to code, to review and to test) than having this code force a prefetch. HasPrefetchedEntirelyZeros() bool + + // WriteBufferTo writes the entire contents of the prefetched buffer to h + // Panics if the internal buffer has not been prefetched (or if its been discarded after a complete Read) + WriteBufferTo(h hash.Hash) } // Simple aggregation of existing io interfaces @@ -118,18 +120,6 @@ func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFa } } -// Prefetch, and ignore any errors (just leave in not-prefetch-yet state, if there was an error) -// If we leave it in the not-prefetched state here, then when Read happens that will trigger another read attempt, -// and that one WILL return any error that happens -func (cr *singleChunkReader) TryBlockingPrefetch(fileReader io.ReaderAt) bool { - err := cr.blockingPrefetch(fileReader, false) - if err != nil { - cr.returnBuffer() // if there was an error, be sure to put us back into a valid "not-yet-prefetched" state - return false - } - return true -} - func (cr *singleChunkReader) HasPrefetchedEntirelyZeros() bool { if cr.buffer == nil { return false // not prefetched (and, to simply error handling in teh caller, we don't call retryBlockingPrefetchIfNecessary here) @@ -153,7 +143,7 @@ func (cr *singleChunkReader) HasPrefetchedEntirelyZeros() bool { // (Allowing the caller to provide the reader to us allows a sequential read approach, since caller can control the order sequentially (in the initial, non-retry, scenario) // We use io.ReaderAt, rather than io.Reader, just for maintainablity/ensuring correctness. (Since just using Reader requires the caller to // follow certain assumptions about positioning the file pointer at the right place before calling us, but using ReaderAt does not). -func (cr *singleChunkReader) blockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { +func (cr *singleChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { if cr.buffer != nil { return nil // already prefetched } @@ -198,7 +188,7 @@ func (cr *singleChunkReader) retryBlockingPrefetchIfNecessary() error { // no need to seek first, because its a ReaderAt const isRetry = true // retries are the only time we need to redo the prefetch - return cr.blockingPrefetch(sourceFile, isRetry) + return cr.BlockingPrefetch(sourceFile, isRetry) } // Seeks within this chunk @@ -307,3 +297,13 @@ func (cr *singleChunkReader) CaptureLeadingBytes() []byte { cr.Seek(0, io.SeekStart) return leadingBytes } + +func (cr *singleChunkReader) WriteBufferTo(h hash.Hash) { + if cr.buffer == nil { + panic("invalid state. No prefetch buffer is present") + } + _, err := h.Write(cr.buffer) + if err != nil { + panic("documentation of hash.Hash.Write says it will never return an error") + } +} diff --git a/ste/JobPartPlan.go b/ste/JobPartPlan.go index e28857d7a..5b86ee8b2 100644 --- a/ste/JobPartPlan.go +++ b/ste/JobPartPlan.go @@ -14,7 +14,7 @@ import ( // dataSchemaVersion defines the data schema version of JobPart order files supported by // current version of azcopy // To be Incremented every time when we release azcopy with changed dataSchema -const DataSchemaVersion common.Version = 1 +const DataSchemaVersion common.Version = 2 const ( ContentTypeMaxBytes = 256 // If > 65536, then jobPartPlanBlobData's ContentTypeLength's type field must change @@ -212,6 +212,9 @@ type JobPartPlanDstLocal struct { // Specifies whether the timestamp of destination file has to be set to the modified time of source file PreserveLastModifiedTime bool + + // says how MD5 verification failures should be actioned + MD5VerificationOption common.HashValidationOption } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/ste/JobPartPlanFileName.go b/ste/JobPartPlanFileName.go index ff0ecb218..5422b843f 100644 --- a/ste/JobPartPlanFileName.go +++ b/ste/JobPartPlanFileName.go @@ -155,6 +155,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { }, DstLocalData: JobPartPlanDstLocal{ PreserveLastModifiedTime: order.BlobAttributes.PreserveLastModifiedTime, + MD5VerificationOption: order.BlobAttributes.MD5ValidationOption, }, atomicJobStatus: common.EJobStatus.InProgress(), // We default to InProgress } @@ -241,7 +242,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { common.PanicIfErr(err) eof += int64(bytesWritten) - // For S2S copy, write the src properties + // For S2S copy (and, in the case of Content-MD5, always), write the src properties if len(order.Transfers[t].ContentType) != 0 { bytesWritten, err = file.WriteString(order.Transfers[t].ContentType) common.PanicIfErr(err) @@ -267,7 +268,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { common.PanicIfErr(err) eof += int64(bytesWritten) } - if order.Transfers[t].ContentMD5 != nil { + if order.Transfers[t].ContentMD5 != nil { // if non-nil but 0 len, will simply not be read by the consumer (since length is zero) bytesWritten, err = file.WriteString(string(order.Transfers[t].ContentMD5)) common.PanicIfErr(err) eof += int64(bytesWritten) diff --git a/ste/md5Comparer.go b/ste/md5Comparer.go new file mode 100644 index 000000000..e7c0ae45a --- /dev/null +++ b/ste/md5Comparer.go @@ -0,0 +1,99 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package ste + +import ( + "bytes" + "errors" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/common" +) + +type transferSpecificLogger interface { + LogAtLevelForCurrentTransfer(level pipeline.LogLevel, msg string) +} + +type md5Comparer struct { + expected []byte + actualAsSaved []byte + validationOption common.HashValidationOption + logger transferSpecificLogger +} + +// TODO: let's add an aka.ms link to the message, that gives more info +var errMd5Mismatch = errors.New("the MD5 hash of the data, as we received it, did not match the expected value, as found in the Blob/File Service. " + + "This means that either there is a data integrity error OR another tool has failed to keep the stored hash up to date") + +// TODO: let's add an aka.ms link to the message, that gives more info +const noMD5Stored = "no MD5 was stored in the Blob/File service against this file. So the downloaded data cannot be MD5-validated." + +var errExpectedMd5Missing = errors.New(noMD5Stored + " This application is currently configured to treat missing MD5 hashes as errors") + +var errActualMd5NotComputed = errors.New("no MDB was computed within this application. This indicates a logic error in this application") + +// Check compares the two MD5s, and returns any error if applicable +// Any informational logging will be done within Check, so all the caller needs to do +// is respond to non-nil errors +func (c *md5Comparer) Check() error { + + if c.actualAsSaved == nil || len(c.actualAsSaved) == 0 { + return errActualMd5NotComputed // Should never happen, so there's no way to opt out of this error being returned if it DOES happen + } + + // missing + if c.expected == nil || len(c.expected) == 0 { + switch c.validationOption { + case common.EHashValidationOption.FailIfDifferentOrMissing(): + return errExpectedMd5Missing + case common.EHashValidationOption.FailIfDifferent(), + common.EHashValidationOption.LogOnly(): + c.logAsMissing() + return nil + default: + panic("unexpected hash validation type") + } + } + + // different + match := bytes.Equal(c.expected, c.actualAsSaved) + if !match { + switch c.validationOption { + case common.EHashValidationOption.FailIfDifferentOrMissing(), + common.EHashValidationOption.FailIfDifferent(): + return errMd5Mismatch + case common.EHashValidationOption.LogOnly(): + c.logAsDifferent() + return nil + default: + panic("unexpected hash validation type") + } + } + + return nil +} + +func (c *md5Comparer) logAsMissing() { + c.logger.LogAtLevelForCurrentTransfer(pipeline.LogWarning, noMD5Stored) +} + +func (c *md5Comparer) logAsDifferent() { + c.logger.LogAtLevelForCurrentTransfer(pipeline.LogWarning, errMd5Mismatch.Error()) +} diff --git a/ste/mgr-JobPartMgr.go b/ste/mgr-JobPartMgr.go index 62e22076a..976e7be15 100644 --- a/ste/mgr-JobPartMgr.go +++ b/ste/mgr-JobPartMgr.go @@ -426,11 +426,11 @@ func (jpm *jobPartMgr) createPipeline(ctx context.Context) { } } -func (jpm *jobPartMgr) SlicePool() common.ByteSlicePooler{ +func (jpm *jobPartMgr) SlicePool() common.ByteSlicePooler { return jpm.slicePool } -func (jpm *jobPartMgr) CacheLimiter() common.CacheLimiter{ +func (jpm *jobPartMgr) CacheLimiter() common.CacheLimiter { return jpm.cacheLimiter } @@ -473,9 +473,8 @@ func (jpm *jobPartMgr) SAS() (string, string) { return jpm.sourceSAS, jpm.destinationSAS } -func (jpm *jobPartMgr) localDstData() (preserveLastModifiedTime bool) { - dstData := &jpm.Plan().DstLocalData - return dstData.PreserveLastModifiedTime +func (jpm *jobPartMgr) localDstData() *JobPartPlanDstLocal { + return &jpm.Plan().DstLocalData } // Call Done when a transfer has completed its epilog; this method returns the number of transfers completed so far @@ -521,11 +520,10 @@ func (jpm *jobPartMgr) ReleaseAConnection() { func (jpm *jobPartMgr) ShouldLog(level pipeline.LogLevel) bool { return jpm.jobMgr.ShouldLog(level) } func (jpm *jobPartMgr) Log(level pipeline.LogLevel, msg string) { jpm.jobMgr.Log(level, msg) } func (jpm *jobPartMgr) Panic(err error) { jpm.jobMgr.Panic(err) } -func (jpm *jobPartMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason){ +func (jpm *jobPartMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason) { jpm.jobMgr.LogChunkStatus(id, reason) } - // TODO: Can we delete this method? // numberOfTransfersDone returns the numberOfTransfersDone_doNotUse of JobPartPlanInfo // instance in thread safe manner diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index 9993dd64c..90057fac5 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -22,6 +22,7 @@ type IJobPartTransferMgr interface { BlobDstData(dataFileToXfer []byte) (headers azblob.BlobHTTPHeaders, metadata azblob.Metadata) FileDstData(dataFileToXfer []byte) (headers azfile.FileHTTPHeaders, metadata azfile.Metadata) PreserveLastModifiedTime() (time.Time, bool) + MD5ValidationOption() common.HashValidationOption BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier) //ScheduleChunk(chunkFunc chunkFunc) Context() context.Context @@ -54,6 +55,7 @@ type IJobPartTransferMgr interface { LogError(resource, context string, err error) LogTransferStart(source, destination, description string) LogChunkStatus(id common.ChunkID, reason common.WaitReason) + LogAtLevelForCurrentTransfer(level pipeline.LogLevel, msg string) common.ILogger } @@ -214,13 +216,17 @@ func (jptm *jobPartTransferMgr) FileDstData(dataFileToXfer []byte) (headers azfi // PreserveLastModifiedTime checks for the PreserveLastModifiedTime flag in JobPartPlan of a transfer. // If PreserveLastModifiedTime is set to true, it returns the lastModifiedTime of the source. func (jptm *jobPartTransferMgr) PreserveLastModifiedTime() (time.Time, bool) { - if preserveLastModifiedTime := jptm.jobPartMgr.(*jobPartMgr).localDstData(); preserveLastModifiedTime { + if preserveLastModifiedTime := jptm.jobPartMgr.(*jobPartMgr).localDstData().PreserveLastModifiedTime; preserveLastModifiedTime { lastModifiedTime := jptm.jobPartPlanTransfer.ModifiedTime return time.Unix(0, lastModifiedTime), true } return time.Time{}, false } +func (jptm *jobPartTransferMgr) MD5ValidationOption() common.HashValidationOption { + return jptm.jobPartMgr.(*jobPartMgr).localDstData().MD5VerificationOption +} + func (jptm *jobPartTransferMgr) BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier) { return jptm.jobPartMgr.BlobTiers() } @@ -381,7 +387,17 @@ const ( transferErrorCodeCopyFailed transferErrorCode = "COPYFAILED" ) +func (jptm *jobPartTransferMgr) LogAtLevelForCurrentTransfer(level pipeline.LogLevel, msg string) { + // order of log elements here is mirrored, with some more added, in logTransferError + fullMsg := common.URLStringExtension(jptm.Info().Source).RedactSigQueryParamForLogging() + " " + + msg + + " Dst: " + common.URLStringExtension(jptm.Info().Destination).RedactSigQueryParamForLogging() + + jptm.Log(level, fullMsg) +} + func (jptm *jobPartTransferMgr) logTransferError(errorCode transferErrorCode, source, destination, errorMsg string, status int) { + // order of log elements here is mirrored, in subset, in LogForCurrentTransfer msg := fmt.Sprintf("%v: ", errorCode) + common.URLStringExtension(source).RedactSigQueryParamForLogging() + fmt.Sprintf(" : %03d : %s\n Dst: ", status, errorMsg) + common.URLStringExtension(destination).RedactSigQueryParamForLogging() jptm.Log(pipeline.LogError, msg) diff --git a/ste/uploader-appendBlob.go b/ste/uploader-appendBlob.go index 61365652d..6cabc0dc0 100644 --- a/ste/uploader-appendBlob.go +++ b/ste/uploader-appendBlob.go @@ -38,6 +38,8 @@ type appendBlobUploader struct { pipeline pipeline.Pipeline pacer *pacer soleChunkFuncSemaphore *semaphore.Weighted + md5Channel chan []byte + creationTimeHeaders *azblob.BlobHTTPHeaders } func newAppendBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -69,6 +71,7 @@ func newAppendBlobUploader(jptm IJobPartTransferMgr, destination string, p pipel pipeline: p, pacer: pacer, soleChunkFuncSemaphore: semaphore.NewWeighted(1), + md5Channel: newMd5Channel(), }, nil } @@ -80,6 +83,10 @@ func (u *appendBlobUploader) NumChunks() uint32 { return u.numChunks } +func (u *appendBlobUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *appendBlobUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.appendBlobUrl.GetProperties(u.jptm.Context(), azblob.BlobAccessConditions{})) } @@ -93,6 +100,8 @@ func (u *appendBlobUploader) Prologue(leadingBytes []byte) { jptm.FailActiveUpload("Creating blob", err) return } + // Save headers to re-use, with same values, in epilogue + u.creationTimeHeaders = &blobHTTPHeaders } func (u *appendBlobUploader) GenerateUploadFunc(id common.ChunkID, blockIndex int32, reader common.SingleChunkReader, chunkIsWholeFile bool) chunkFunc { @@ -132,6 +141,17 @@ func (u *appendBlobUploader) GenerateUploadFunc(id common.ChunkID, blockIndex in func (u *appendBlobUploader) Epilogue() { jptm := u.jptm + + // set content MD5 (only way to do this is to re-PUT all the headers, this time with the MD5 included) + if jptm.TransferStatus() > 0 { + tryPutMd5Hash(jptm, u.md5Channel, func(md5Hash []byte) error { + epilogueHeaders := *u.creationTimeHeaders + epilogueHeaders.ContentMD5 = md5Hash + _, err := u.appendBlobUrl.SetHTTPHeaders(jptm.Context(), epilogueHeaders, azblob.BlobAccessConditions{}) + return err + }) + } + // Cleanup if jptm.TransferStatus() <= 0 { // TODO: <=0 or <0? // If the transfer status value < 0, then transfer failed with some failure diff --git a/ste/uploader-azureFiles.go b/ste/uploader-azureFiles.go index c6418a608..6aebd98bb 100644 --- a/ste/uploader-azureFiles.go +++ b/ste/uploader-azureFiles.go @@ -34,12 +34,14 @@ import ( ) type azureFilesUploader struct { - jptm IJobPartTransferMgr - fileURL azfile.FileURL - chunkSize uint32 - numChunks uint32 - pipeline pipeline.Pipeline - pacer *pacer + jptm IJobPartTransferMgr + fileURL azfile.FileURL + chunkSize uint32 + numChunks uint32 + pipeline pipeline.Pipeline + pacer *pacer + md5Channel chan []byte + creationTimeHeaders *azfile.FileHTTPHeaders // pointer so default value, nil, is clearly "wrong" and can't be used by accident } func newAzureFilesUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -68,12 +70,13 @@ func newAzureFilesUploader(jptm IJobPartTransferMgr, destination string, p pipel } return &azureFilesUploader{ - jptm: jptm, - fileURL: azfile.NewFileURL(*destURL, p), - chunkSize: chunkSize, - numChunks: numChunks, - pipeline: p, - pacer: pacer, + jptm: jptm, + fileURL: azfile.NewFileURL(*destURL, p), + chunkSize: chunkSize, + numChunks: numChunks, + pipeline: p, + pacer: pacer, + md5Channel: newMd5Channel(), }, nil } @@ -85,6 +88,10 @@ func (u *azureFilesUploader) NumChunks() uint32 { return u.numChunks } +func (u *azureFilesUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *azureFilesUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.fileURL.GetProperties(u.jptm.Context())) } @@ -107,6 +114,9 @@ func (u *azureFilesUploader) Prologue(leadingBytes []byte) { jptm.FailActiveUpload("Creating file", err) return } + + // Save headers to re-use, with same values, in epilogue + u.creationTimeHeaders = &fileHTTPHeaders } func (u *azureFilesUploader) GenerateUploadFunc(id common.ChunkID, blockIndex int32, reader common.SingleChunkReader, chunkIsWholeFile bool) chunkFunc { @@ -141,6 +151,16 @@ func (u *azureFilesUploader) GenerateUploadFunc(id common.ChunkID, blockIndex in func (u *azureFilesUploader) Epilogue() { jptm := u.jptm + // set content MD5 (only way to do this is to re-PUT all the headers, this time with the MD5 included) + if jptm.TransferStatus() > 0 { + tryPutMd5Hash(jptm, u.md5Channel, func(md5Hash []byte) error { + epilogueHeaders := *u.creationTimeHeaders + epilogueHeaders.ContentMD5 = md5Hash + _, err := u.fileURL.SetHTTPHeaders(jptm.Context(), epilogueHeaders) + return err + }) + } + // Cleanup if jptm.TransferStatus() <= 0 { // If the transfer status is less than or equal to 0 diff --git a/ste/uploader-blobFS.go b/ste/uploader-blobFS.go index 91b6b6098..69c0fe624 100644 --- a/ste/uploader-blobFS.go +++ b/ste/uploader-blobFS.go @@ -32,12 +32,13 @@ import ( ) type blobFSUploader struct { - jptm IJobPartTransferMgr - fileURL azbfs.FileURL - chunkSize uint32 - numChunks uint32 - pipeline pipeline.Pipeline - pacer *pacer + jptm IJobPartTransferMgr + fileURL azbfs.FileURL + chunkSize uint32 + numChunks uint32 + pipeline pipeline.Pipeline + pacer *pacer + md5Channel chan []byte } func newBlobFSUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -105,12 +106,13 @@ func newBlobFSUploader(jptm IJobPartTransferMgr, destination string, p pipeline. numChunks := getNumUploadChunks(info.SourceSize, chunkSize) return &blobFSUploader{ - jptm: jptm, - fileURL: azbfs.NewFileURL(*destURL, p), - chunkSize: chunkSize, - numChunks: numChunks, - pipeline: p, - pacer: pacer, + jptm: jptm, + fileURL: azbfs.NewFileURL(*destURL, p), + chunkSize: chunkSize, + numChunks: numChunks, + pipeline: p, + pacer: pacer, + md5Channel: newMd5Channel(), }, nil } @@ -122,6 +124,11 @@ func (u *blobFSUploader) NumChunks() uint32 { return u.numChunks } +func (u *blobFSUploader) Md5Channel() chan<- []byte { + // TODO: can we support this? And when? Right now, we are returning it, but never using it ourselves + return u.md5Channel +} + func (u *blobFSUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.fileURL.GetProperties(u.jptm.Context())) } diff --git a/ste/uploader-blockBlob.go b/ste/uploader-blockBlob.go index 91c2f89cd..40b0a5c32 100644 --- a/ste/uploader-blockBlob.go +++ b/ste/uploader-blockBlob.go @@ -44,6 +44,7 @@ type blockBlobUploader struct { leadingBytes []byte // no lock because is written before first chunk-func go routine is scheduled mu *sync.Mutex // protects the fields below blockIds []string + md5Channel chan []byte } func newBlockBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -74,6 +75,7 @@ func newBlockBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeli pacer: pacer, mu: &sync.Mutex{}, blockIds: make([]string, numChunks), + md5Channel: newMd5Channel(), }, nil } @@ -85,6 +87,10 @@ func (u *blockBlobUploader) NumChunks() uint32 { return u.numChunks } +func (u *blockBlobUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *blockBlobUploader) SetLeadingBytes(leadingBytes []byte) { u.leadingBytes = leadingBytes } @@ -151,8 +157,21 @@ func (u *blockBlobUploader) generatePutWholeBlob(id common.ChunkID, blockIndex i jptm.LogChunkStatus(id, common.EWaitReason.Body()) var err error if jptm.Info().SourceSize == 0 { + // Empty file _, err = u.blockBlobUrl.Upload(jptm.Context(), bytes.NewReader(nil), blobHttpHeader, metaData, azblob.BlobAccessConditions{}) + } else { + // File with content + + // Get the MD5 that was computed as we read the file + md5Hash, ok := <-u.md5Channel + if !ok { + jptm.FailActiveUpload("Getting hash", errNoHash) + return + } + blobHttpHeader.ContentMD5 = md5Hash + + // Upload the file body := newLiteRequestBodyPacer(reader, u.pacer) _, err = u.blockBlobUrl.Upload(jptm.Context(), body, blobHttpHeader, metaData, azblob.BlobAccessConditions{}) } @@ -182,13 +201,20 @@ func (u *blockBlobUploader) Epilogue() { if jptm.TransferStatus() > 0 && shouldPutBlockList == putListNeeded { jptm.Log(pipeline.LogDebug, fmt.Sprintf("Conclude Transfer with BlockList %s", blockIds)) - // fetching the blob http headers with content-type, content-encoding attributes - // fetching the metadata passed with the JobPartOrder - blobHttpHeader, metaData := jptm.BlobDstData(u.leadingBytes) + md5Hash, ok := <-u.md5Channel + if ok { + // fetching the blob http headers with content-type, content-encoding attributes + // fetching the metadata passed with the JobPartOrder + blobHttpHeader, metaData := jptm.BlobDstData(u.leadingBytes) + blobHttpHeader.ContentMD5 = md5Hash - _, err := u.blockBlobUrl.CommitBlockList(jptm.Context(), blockIds, blobHttpHeader, metaData, azblob.BlobAccessConditions{}) - if err != nil { - jptm.FailActiveUpload("Committing block list", err) + _, err := u.blockBlobUrl.CommitBlockList(jptm.Context(), blockIds, blobHttpHeader, metaData, azblob.BlobAccessConditions{}) + if err != nil { + jptm.FailActiveUpload("Committing block list", err) + // don't return, since need cleanup below + } + } else { + jptm.FailActiveUpload("Getting hash", errNoHash) // don't return, since need cleanup below } } diff --git a/ste/uploader-pageBlob.go b/ste/uploader-pageBlob.go index 6c504ea09..fea651fbf 100644 --- a/ste/uploader-pageBlob.go +++ b/ste/uploader-pageBlob.go @@ -31,12 +31,14 @@ import ( ) type pageBlobUploader struct { - jptm IJobPartTransferMgr - pageBlobUrl azblob.PageBlobURL - chunkSize uint32 - numChunks uint32 - pipeline pipeline.Pipeline - pacer *pacer + jptm IJobPartTransferMgr + pageBlobUrl azblob.PageBlobURL + chunkSize uint32 + numChunks uint32 + pipeline pipeline.Pipeline + pacer *pacer + md5Channel chan []byte + creationTimeHeaders *azblob.BlobHTTPHeaders } func newPageBlobUploader(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) { @@ -65,6 +67,7 @@ func newPageBlobUploader(jptm IJobPartTransferMgr, destination string, p pipelin numChunks: numChunks, pipeline: p, pacer: pacer, + md5Channel: newMd5Channel(), }, nil } @@ -76,6 +79,10 @@ func (u *pageBlobUploader) NumChunks() uint32 { return u.numChunks } +func (u *pageBlobUploader) Md5Channel() chan<- []byte { + return u.md5Channel +} + func (u *pageBlobUploader) RemoteFileExists() (bool, error) { return remoteObjectExists(u.pageBlobUrl.GetProperties(u.jptm.Context(), azblob.BlobAccessConditions{})) } @@ -92,6 +99,8 @@ func (u *pageBlobUploader) Prologue(leadingBytes []byte) { jptm.FailActiveUpload("Creating blob", err) return } + // Save headers to re-use, with same values, in epilogue + u.creationTimeHeaders = &blobHTTPHeaders // set tier _, pageBlobTier := jptm.BlobTiers() @@ -137,6 +146,16 @@ func (u *pageBlobUploader) GenerateUploadFunc(id common.ChunkID, blockIndex int3 func (u *pageBlobUploader) Epilogue() { jptm := u.jptm + // set content MD5 (only way to do this is to re-PUT all the headers, this time with the MD5 included) + if jptm.TransferStatus() > 0 { + tryPutMd5Hash(jptm, u.md5Channel, func(md5Hash []byte) error { + epilogueHeaders := *u.creationTimeHeaders + epilogueHeaders.ContentMD5 = md5Hash + _, err := u.pageBlobUrl.SetHTTPHeaders(jptm.Context(), epilogueHeaders, azblob.BlobAccessConditions{}) + return err + }) + } + // Cleanup if jptm.TransferStatus() <= 0 { // TODO: <=0 or <0? deletionContext, cancelFn := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/ste/uploader.go b/ste/uploader.go index 58ba4318f..57a00dd83 100644 --- a/ste/uploader.go +++ b/ste/uploader.go @@ -21,6 +21,7 @@ package ste import ( + "errors" "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" ) @@ -58,10 +59,35 @@ type uploader interface { // or post-failure processing otherwise. Implementations should // use jptm.FailActiveUpload if anything fails during the epilogue. Epilogue() + + // Md5Channel returns the channel on which localToRemote should send the MD5 hash to the uploader + Md5Channel() chan<- []byte } type uploaderFactory func(jptm IJobPartTransferMgr, destination string, p pipeline.Pipeline, pacer *pacer) (uploader, error) +func newMd5Channel() chan []byte { + return make(chan []byte, 1) // must be buffered, so as not to hold up the goroutine running localToRemote (which needs to start on the NEXT file after finishing its current one) +} + +// Tries to set the MD5 hash using the given function +// Fails the upload if any error happens. +// This should be used only by those uploads that require a separate operation to PUT the hash at the end. +// Others, such as the block blob uploader piggyback their MD5 setting on other calls, and so won't use this. +func tryPutMd5Hash(jptm IJobPartTransferMgr, md5Channel <-chan []byte, worker func(hash []byte) error) { + md5Hash, ok := <-md5Channel + if ok { + err := worker(md5Hash) + if err != nil { + jptm.FailActiveUpload("Setting hash", err) + } + } else { + jptm.FailActiveUpload("Setting hash", errNoHash) + } +} + +var errNoHash = errors.New("no hash computed") + func getNumUploadChunks(fileSize int64, chunkSize uint32) uint32 { numChunks := uint32(1) // for uploads, we always map zero-size files to ONE (empty) chunk if fileSize > 0 { diff --git a/ste/xfer-localToRemote.go b/ste/xfer-localToRemote.go index 9050b44df..d0104acf1 100644 --- a/ste/xfer-localToRemote.go +++ b/ste/xfer-localToRemote.go @@ -21,6 +21,7 @@ package ste import ( + "crypto/md5" "fmt" "os" @@ -48,13 +49,14 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, jptm.ReportTransferDone() return } - // step 2b. Read chunk size and count from the uploader (since it may have applied its own defaults and/or calculations to produce these values - chunkSize := ul.ChunkSize() - numChunks := ul.NumChunks() + md5Channel := ul.Md5Channel() + defer close(md5Channel) // never leave receiver hanging, waiting for a result, even if we fail here + + // step 2b. Check chunk size and count from the uploader (it may have applied its own defaults and/or calculations to produce these values if jptm.ShouldLog(pipeline.LogInfo) { - jptm.LogTransferStart(info.Source, info.Destination, fmt.Sprintf("Specified chunk size %d", chunkSize)) + jptm.LogTransferStart(info.Source, info.Destination, fmt.Sprintf("Specified chunk size %d", ul.ChunkSize())) } - if numChunks == 0 { + if ul.NumChunks() == 0 { panic("must always schedule one chunk, even if file is empty") // this keeps our code structure simpler, by using a dummy chunk for empty files } @@ -102,7 +104,7 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, // ****** // step 5: tell jptm what to expect, and how to clean up at the end - jptm.SetNumberOfChunks(numChunks) + jptm.SetNumberOfChunks(ul.NumChunks()) jptm.SetActionAfterLastChunk(func() { epilogueWithCleanupUpload(jptm, ul) }) // TODO: currently, the epilogue will only run if the number of completed chunks = numChunks. @@ -114,20 +116,30 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, // eventually reach numChunks, since we have no better short-term alternative. // Step 5: Go through the file and schedule chunk messages to upload each chunk - // As we do this, we force preload of each chunk to memory, and we wait (block) - // here if the amount of preloaded data gets excessive. That's OK to do, - // because if we already have that much data preloaded (and scheduled for sending in - // chunks) then we don't need to schedule any more chunks right now, so the blocking - // is harmless (and a good thing, to avoid excessive RAM usage). - // To take advantage of the good sequential read performance provided by many file systems, - // we work sequentially through the file here. + scheduleUploadChunks(jptm, info.Source, srcFile, fileSize, ul, sourceFileFactory, md5Channel) +} + +// Schedule all the upload chunks. +// As we do this, we force preload of each chunk to memory, and we wait (block) +// here if the amount of preloaded data gets excessive. That's OK to do, +// because if we already have that much data preloaded (and scheduled for sending in +// chunks) then we don't need to schedule any more chunks right now, so the blocking +// is harmless (and a good thing, to avoid excessive RAM usage). +// To take advantage of the good sequential read performance provided by many file systems, +// and to be able to compute an MD5 hash for the file, we work sequentially through the file here. +func scheduleUploadChunks(jptm IJobPartTransferMgr, srcName string, srcFile common.CloseableReaderAt, fileSize int64, ul uploader, sourceFileFactory common.ChunkReaderSourceFactory, md5Channel chan<- []byte) { + chunkSize := ul.ChunkSize() + numChunks := ul.NumChunks() context := jptm.Context() slicePool := jptm.SlicePool() cacheLimiter := jptm.CacheLimiter() + chunkCount := int32(0) + md5Hasher := md5.New() + safeToUseHash := true for startIndex := int64(0); startIndex < fileSize || isDummyChunkInEmptyFile(startIndex, fileSize); startIndex += int64(chunkSize) { - id := common.ChunkID{Name: info.Source, OffsetInFile: startIndex} + id := common.ChunkID{Name: srcName, OffsetInFile: startIndex} adjustedChunkSize := int64(chunkSize) // compute actual size of the chunk @@ -143,7 +155,20 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, chunkReader := common.NewSingleChunkReader(context, sourceFileFactory, id, adjustedChunkSize, jptm, slicePool, cacheLimiter) // Wait until we have enough RAM, and when we do, prefetch the data for this chunk. - chunkReader.TryBlockingPrefetch(srcFile) + chunkDataError := chunkReader.BlockingPrefetch(srcFile, false) + + // Add the bytes to the hash + // NOTE: if there is a retry on this chunk later (a 503 from Service) our current implementation of singleChunkReader + // (as at Jan 2019) will re-read from the disk. If that part of the file has been updated by another process, + // that means it will not longer match the hash we set here. That would be bad. So we rely on logic + // elsewhere in our upload code to avoid/fail or retry such transfers. + // TODO: move the above note to the place where we implement the avoid/fail/retry and refer to that in a comment + // on the retry file-re-read logic + if chunkDataError == nil { + chunkReader.WriteBufferTo(md5Hasher) + } else { + safeToUseHash = false // because we've missed a chunk + } // If this is the the very first chunk, do special init steps if startIndex == 0 { @@ -158,15 +183,27 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, // schedule the chunk job/msg jptm.LogChunkStatus(id, common.EWaitReason.WorkerGR()) - isWholeFile := numChunks == 1 - jptm.ScheduleChunks(ul.GenerateUploadFunc(id, chunkCount, chunkReader, isWholeFile)) + var cf chunkFunc + if chunkDataError == nil { + isWholeFile := numChunks == 1 + cf = ul.GenerateUploadFunc(id, chunkCount, chunkReader, isWholeFile) + } else { + _ = chunkReader.Close() + // Our jptm logic currently requires us to schedule every chunk, even if we know there's an error, + // so we schedule a func that will just fail with the given error + cf = createUploadChunkFunc(jptm, id, func() { jptm.FailActiveUpload("chunk data read", chunkDataError) }) + } + jptm.ScheduleChunks(cf) chunkCount += 1 } - // sanity check to verify the number of chunks scheduled if chunkCount != int32(numChunks) { - panic(fmt.Errorf("difference in the number of chunk calculated %v and actual chunks scheduled %v for src %s of size %v", numChunks, chunkCount, info.Source, fileSize)) + panic(fmt.Errorf("difference in the number of chunk calculated %v and actual chunks scheduled %v for src %s of size %v", numChunks, chunkCount, srcName, fileSize)) + } + // provide the hash that we computed + if safeToUseHash { + md5Channel <- md5Hasher.Sum(nil) } } diff --git a/ste/xfer-remoteToLocal.go b/ste/xfer-remoteToLocal.go index 684eff11a..b30991ccd 100644 --- a/ste/xfer-remoteToLocal.go +++ b/ste/xfer-remoteToLocal.go @@ -81,7 +81,7 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, if err != nil { jptm.LogDownloadError(info.Source, info.Destination, "File Creation Error "+err.Error(), 0) jptm.SetStatus(common.ETransferStatus.Failed()) - epilogueWithCleanupDownload(jptm, nil, nil) + epilogueWithCleanupDownload(jptm, nil, nil) // use standard epilogue for consistency return } // TODO: Question: do we need to Stat the file, to check its size, after explicitly making it with the desired size? @@ -170,22 +170,30 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, func epilogueWithCleanupDownload(jptm IJobPartTransferMgr, activeDstFile *os.File, cw common.ChunkedFileWriter) { info := jptm.Info() - if activeDstFile != nil { + haveNonEmptyFile := activeDstFile != nil + if haveNonEmptyFile { + // wait until all received chunks are flushed out - _, flushError := cw.Flush(jptm.Context()) // todo: use, and check the MD5 hash returned here + md5OfFileAsWritten, flushError := cw.Flush(jptm.Context()) + closeErr := activeDstFile.Close() // always try to close if, even if flush failed + if flushError != nil { + jptm.FailActiveDownload("Flushing file", flushError) + } + if closeErr != nil { + jptm.FailActiveDownload("Closing file", closeErr) + } - // Close file - fileCloseErr := activeDstFile.Close() // always try to close if, even if flush failed - if (flushError != nil || fileCloseErr != nil) && !jptm.TransferStatus().DidFail() { - // it WAS successful up to now, but the file flush/closing failed. - message := "" - if flushError != nil { - message = "File Flush Error " + flushError.Error() - } else { - message = "File Closure Error " + fileCloseErr.Error() + // Check MD5 (but only if file was fully flushed and saved - else no point and may not have actualAsSaved hash anyway) + if !jptm.TransferStatus().DidFail() { + comparison := md5Comparer{ + expected: info.SrcHTTPHeaders.ContentMD5, // the MD5 that came back from Service when we enumerated the source + actualAsSaved: md5OfFileAsWritten, + validationOption: jptm.MD5ValidationOption(), + logger: jptm} + err := comparison.Check() + if err != nil { + jptm.FailActiveDownload("Checking MD5 hash", err) } - jptm.LogDownloadError(info.Source, info.Destination, message, 0) - jptm.SetStatus(common.ETransferStatus.Failed()) } } From d3f2fdf6f6b8cf19219e598efeed502bd4baa918 Mon Sep 17 00:00:00 2001 From: John Rusk Date: Sat, 9 Feb 2019 14:38:56 +1300 Subject: [PATCH 07/64] Extra dignostics for chunk reader out of bounds error (#204) * Extra diagnostics for slice out of bounds error * Add range check to int cast * Panic if multiple public methods of singleChunkReader are invoked at the same time As diagnostic for GitHub issue 191. Testing shows that any CPU usage overhead of this is small enough to be hard to detect. * Improve logging of chunk early close * Tidy logging of callstacks for early Close of singleChunkreader * Trigger CI --- common/singleChunkReader.go | 115 ++++++++++++++++++++++++++++++++++-- ste/xfer-localToRemote.go | 2 +- 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/common/singleChunkReader.go b/common/singleChunkReader.go index ac0d45a6a..39761a677 100644 --- a/common/singleChunkReader.go +++ b/common/singleChunkReader.go @@ -21,10 +21,15 @@ package common import ( + "bytes" "context" "errors" + "github.com/Azure/azure-pipeline-go/pipeline" "hash" "io" + "math" + "runtime" + "sync/atomic" ) // Reader of ONE chunk of a file. Maybe used to re-read multiple times (e.g. if @@ -75,6 +80,9 @@ type CloseableReaderAt interface { type ChunkReaderSourceFactory func() (CloseableReaderAt, error) type singleChunkReader struct { + // for diagnostics for issue https://github.com/Azure/azure-storage-azcopy/issues/191 + atomicUseIndicator int32 + // context used to allow cancellation of blocking operations // (Yes, ideally contexts are not stored in structs, but we need it inside Read, and there's no way for it to be passed in there) ctx context.Context @@ -88,6 +96,9 @@ type singleChunkReader struct { // for logging chunk state transitions chunkLogger ChunkStatusLogger + // general-purpose logger + generalLogger ILogger + // A factory to get hold of the file, in case we need to re-read any of it sourceFactory ChunkReaderSourceFactory @@ -105,13 +116,14 @@ type singleChunkReader struct { // TODO: pooling of buffers to reduce pressure on GC? } -func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFactory, chunkId ChunkID, length int64, chunkLogger ChunkStatusLogger, slicePool ByteSlicePooler, cacheLimiter CacheLimiter) SingleChunkReader { +func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFactory, chunkId ChunkID, length int64, chunkLogger ChunkStatusLogger, generalLogger ILogger, slicePool ByteSlicePooler, cacheLimiter CacheLimiter) SingleChunkReader { if length <= 0 { return &emptyChunkReader{} } return &singleChunkReader{ ctx: ctx, chunkLogger: chunkLogger, + generalLogger: generalLogger, slicePool: slicePool, cacheLimiter: cacheLimiter, sourceFactory: sourceFactory, @@ -120,9 +132,28 @@ func NewSingleChunkReader(ctx context.Context, sourceFactory ChunkReaderSourceFa } } +// Use and un-use are temporary, for identifying the root cause of +// https://github.com/Azure/azure-storage-azcopy/issues/191 +// They may be removed after that. +// For now, they are used to wrap every Public method +func (cr *singleChunkReader) use() { + if atomic.SwapInt32(&cr.atomicUseIndicator, 1) != 0 { + panic("trying to use chunk reader when already in use") + } +} + +func (cr *singleChunkReader) unuse() { + if atomic.SwapInt32(&cr.atomicUseIndicator, 0) != 1 { + panic("ending use when chunk reader was not actually IN use") + } +} + func (cr *singleChunkReader) HasPrefetchedEntirelyZeros() bool { + cr.use() + defer cr.unuse() + if cr.buffer == nil { - return false // not prefetched (and, to simply error handling in teh caller, we don't call retryBlockingPrefetchIfNecessary here) + return false // not prefetched (and, to simply error handling in teh caller, we don't call retryBlockingPrefetchIfNecessary here) } for _, b := range cr.buffer { @@ -139,11 +170,18 @@ func (cr *singleChunkReader) HasPrefetchedEntirelyZeros() bool { // and (c) we would want to check whether it really did offer meaningful real-world performance gain, before introducing use of unsafe. } +func (cr *singleChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { + cr.use() + defer cr.unuse() + + return cr.blockingPrefetch(fileReader, isRetry) +} + // Prefetch the data in this chunk, using a file reader that is provided to us. // (Allowing the caller to provide the reader to us allows a sequential read approach, since caller can control the order sequentially (in the initial, non-retry, scenario) // We use io.ReaderAt, rather than io.Reader, just for maintainablity/ensuring correctness. (Since just using Reader requires the caller to // follow certain assumptions about positioning the file pointer at the right place before calling us, but using ReaderAt does not). -func (cr *singleChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { +func (cr *singleChunkReader) blockingPrefetch(fileReader io.ReaderAt, isRetry bool) error { if cr.buffer != nil { return nil // already prefetched } @@ -159,7 +197,7 @@ func (cr *singleChunkReader) BlockingPrefetch(fileReader io.ReaderAt, isRetry bo } // get buffer from pool - cr.buffer = cr.slicePool.RentSlice(uint32(cr.length)) + cr.buffer = cr.slicePool.RentSlice(uint32Checked(cr.length)) // read bytes into the buffer cr.chunkLogger.LogChunkStatus(cr.chunkId, EWaitReason.Disk()) @@ -188,12 +226,14 @@ func (cr *singleChunkReader) retryBlockingPrefetchIfNecessary() error { // no need to seek first, because its a ReaderAt const isRetry = true // retries are the only time we need to redo the prefetch - return cr.BlockingPrefetch(sourceFile, isRetry) + return cr.blockingPrefetch(sourceFile, isRetry) } // Seeks within this chunk // Seeking is used for retries, and also by some code to get length (by seeking to end) func (cr *singleChunkReader) Seek(offset int64, whence int) (int64, error) { + cr.use() + defer cr.unuse() newPosition := cr.positionInChunk @@ -219,6 +259,9 @@ func (cr *singleChunkReader) Seek(offset int64, whence int) (int64, error) { // Reads from within this chunk func (cr *singleChunkReader) Read(p []byte) (n int, err error) { + cr.use() + defer cr.unuse() + // This is a normal read, so free the prefetch buffer when hit EOF (i.e. end of this chunk). // We do so on the assumption that if we've read to the end we don't need the prefetched data any longer. // (If later, there's a retry that forces seek back to start and re-read, we'll automatically trigger a re-fetch at that time) @@ -240,6 +283,17 @@ func (cr *singleChunkReader) doRead(p []byte, freeBufferOnEof bool) (n int, err return 0, err } + // extra checks until we find root cause of https://github.com/Azure/azure-storage-azcopy/issues/191 + if cr.buffer == nil { + panic("unexpected nil buffer") + } + if cr.positionInChunk >= cr.length { + panic("unexpected EOF") + } + if cr.length != int64(len(cr.buffer)) { + panic("unexpected buffer length discrepancy") + } + // Copy the data across bytesCopied := copy(p, cr.buffer[cr.positionInChunk:]) cr.positionInChunk += int64(bytesCopied) @@ -266,6 +320,9 @@ func (cr *singleChunkReader) returnBuffer() { } func (cr *singleChunkReader) Length() int64 { + cr.use() + defer cr.unuse() + return cr.length } @@ -275,6 +332,26 @@ func (cr *singleChunkReader) Length() int64 { // Without this close, if something failed part way through, we would keep counting this object's bytes in cacheLimiter // "for ever", even after the object is gone. func (cr *singleChunkReader) Close() error { + // first, check and log early closes (before we do use(), since the situation we are trying + // to log is suspected to be one when use() will panic) + if cr.positionInChunk < cr.length { + // this is an "early close". Adjust logging verbosity depending on whether context is still active + var extraMessage string + if cr.ctx.Err() == nil { + b := &bytes.Buffer{} + b.Write(stack()) + extraMessage = "context active so logging full callstack, as follows: " + b.String() + } else { + extraMessage = "context cancelled so no callstack logged" + } + cr.generalLogger.Log(pipeline.LogInfo, "Early close of chunk in singleChunkReader: "+extraMessage) + } + + // after logging callstack, do normal use() + cr.use() + defer cr.unuse() + + // do the real work cr.returnBuffer() return nil } @@ -283,22 +360,31 @@ func (cr *singleChunkReader) Close() error { // (else we would have to re-read the start of the file later, and that breaks our rule to use sequential // reads as much as possible) func (cr *singleChunkReader) CaptureLeadingBytes() []byte { + cr.use() + // can't defer unuse here. See explict calls (plural) below + const mimeRecgonitionLen = 512 leadingBytes := make([]byte, mimeRecgonitionLen) n, err := cr.doRead(leadingBytes, false) // do NOT free bufferOnEOF. So that if its a very small file, and we hit the end, we won't needlessly discard the prefetched data if err != nil && err != io.EOF { + cr.unuse() return nil // we just can't sniff the mime type } if n < len(leadingBytes) { // truncate if we read less than expected (very small file, so err was EOF above) leadingBytes = leadingBytes[:n] } + // unuse before Seek, since Seek is public + cr.unuse() // MUST re-wind, so that the bytes we read will get transferred too! cr.Seek(0, io.SeekStart) return leadingBytes } func (cr *singleChunkReader) WriteBufferTo(h hash.Hash) { + cr.use() + defer cr.unuse() + if cr.buffer == nil { panic("invalid state. No prefetch buffer is present") } @@ -307,3 +393,22 @@ func (cr *singleChunkReader) WriteBufferTo(h hash.Hash) { panic("documentation of hash.Hash.Write says it will never return an error") } } + +func stack() []byte { + buf := make([]byte, 2048) + for { + n := runtime.Stack(buf, false) + if n < len(buf) { + return buf[:n] + } + buf = make([]byte, 2*len(buf)) + } +} + +// while we never expect any out of range errors, due to chunk sizes fitting easily into uint32, here we make sure +func uint32Checked(i int64) uint32 { + if i > math.MaxUint32 { + panic("int64 out of range for cast to uint32") + } + return uint32(i) +} diff --git a/ste/xfer-localToRemote.go b/ste/xfer-localToRemote.go index d0104acf1..ce1e1a9de 100644 --- a/ste/xfer-localToRemote.go +++ b/ste/xfer-localToRemote.go @@ -152,7 +152,7 @@ func scheduleUploadChunks(jptm IJobPartTransferMgr, srcName string, srcFile comm // of the file read later (when doing a retry) // BTW, the reader we create here just works with a single chuck. (That's in contrast with downloads, where we have // to use an object that encompasses the whole file, so that it can put the chunks back into order. We don't have that requirement here.) - chunkReader := common.NewSingleChunkReader(context, sourceFileFactory, id, adjustedChunkSize, jptm, slicePool, cacheLimiter) + chunkReader := common.NewSingleChunkReader(context, sourceFileFactory, id, adjustedChunkSize, jptm, jptm, slicePool, cacheLimiter) // Wait until we have enough RAM, and when we do, prefetch the data for this chunk. chunkDataError := chunkReader.BlockingPrefetch(srcFile, false) From 6aef9062de9496eafae9198ae3d834ba639e1317 Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Tue, 5 Feb 2019 10:01:41 -0800 Subject: [PATCH 08/64] Job completion info is more descriptive to avoid overlooking errors --- cmd/copy.go | 2 +- cmd/jobsResume.go | 2 +- cmd/sync.go | 2 +- common/fe-ste-models.go | 36 ++++++++++++-- common/fe-ste-models_test.go | 91 ++++++++++++++++++++++++++++++++++++ common/zt_credCache_test.go | 7 ++- go.mod | 7 ++- go.sum | 14 +++--- ste/init.go | 13 ++++++ 9 files changed, 155 insertions(+), 19 deletions(-) create mode 100644 common/fe-ste-models_test.go diff --git a/cmd/copy.go b/cmd/copy.go index d5e342e22..60ca5c362 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -761,7 +761,7 @@ func (cca *cookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { // fetch a job status var summary common.ListJobSummaryResponse Rpc(common.ERpcCmd.ListJobSummary(), &cca.jobID, &summary) - jobDone := summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() + jobDone := summary.JobStatus.IsJobDone() // if json output is desired, simply marshal and return // note that if job is already done, we simply exit diff --git a/cmd/jobsResume.go b/cmd/jobsResume.go index f8cfa8574..b70b03204 100644 --- a/cmd/jobsResume.go +++ b/cmd/jobsResume.go @@ -82,7 +82,7 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) { // fetch a job status var summary common.ListJobSummaryResponse Rpc(common.ERpcCmd.ListJobSummary(), &cca.jobID, &summary) - jobDone := summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() + jobDone := summary.JobStatus.IsJobDone() // if json is not desired, and job is done, then we generate a special end message to conclude the job if jobDone { diff --git a/cmd/sync.go b/cmd/sync.go index 72649f279..6de690040 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -259,7 +259,7 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { // fetch a job status var summary common.ListSyncJobSummaryResponse Rpc(common.ERpcCmd.ListSyncJobSummary(), &cca.jobID, &summary) - jobDone := summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() + jobDone := summary.JobStatus.IsJobDone() // if json output is desired, simply marshal and return // note that if job is already done, we simply exit diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 2bff1d6b2..b4ac37c40 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -218,11 +218,37 @@ func (j *JobStatus) AtomicStore(newJobStatus JobStatus) { atomic.StoreUint32((*uint32)(j), uint32(newJobStatus)) } -func (JobStatus) InProgress() JobStatus { return JobStatus(0) } -func (JobStatus) Paused() JobStatus { return JobStatus(1) } -func (JobStatus) Cancelling() JobStatus { return JobStatus(2) } -func (JobStatus) Cancelled() JobStatus { return JobStatus(3) } -func (JobStatus) Completed() JobStatus { return JobStatus(4) } +func (j *JobStatus) EnhanceJobStatusInfo(skippedTransfers, failedTransfers, successfulTransfers bool) JobStatus { + if failedTransfers && skippedTransfers { + return EJobStatus.CompletedWithErrorsAndSkipped() + } else if failedTransfers { + if successfulTransfers { + return EJobStatus.CompletedWithErrors() + } else { + return EJobStatus.Failed() + } + } else if skippedTransfers { + return EJobStatus.CompletedWithSkipped() + } else { + return EJobStatus.Completed() + } +} + +func (j *JobStatus) IsJobDone() bool { + return *j == EJobStatus.Completed() || *j == EJobStatus.Cancelled() || *j == EJobStatus.CompletedWithSkipped() || + *j == EJobStatus.CompletedWithErrors() || *j == EJobStatus.CompletedWithErrorsAndSkipped() || + *j == EJobStatus.Failed() +} + +func (JobStatus) InProgress() JobStatus { return JobStatus(0) } +func (JobStatus) Paused() JobStatus { return JobStatus(1) } +func (JobStatus) Cancelling() JobStatus { return JobStatus(2) } +func (JobStatus) Cancelled() JobStatus { return JobStatus(3) } +func (JobStatus) Completed() JobStatus { return JobStatus(4) } +func (JobStatus) CompletedWithErrors() JobStatus { return JobStatus(5) } +func (JobStatus) CompletedWithSkipped() JobStatus { return JobStatus(6) } +func (JobStatus) CompletedWithErrorsAndSkipped() JobStatus { return JobStatus(7) } +func (JobStatus) Failed() JobStatus { return JobStatus(8) } func (js JobStatus) String() string { return enum.StringInt(js, reflect.TypeOf(js)) } diff --git a/common/fe-ste-models_test.go b/common/fe-ste-models_test.go new file mode 100644 index 000000000..39e77f547 --- /dev/null +++ b/common/fe-ste-models_test.go @@ -0,0 +1,91 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package common_test + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" +) + +type feSteModelsTestSuite struct{} + +var _ = chk.Suite(&feSteModelsTestSuite{}) + +func (s *feSteModelsTestSuite) TestEnhanceJobStatusInfo(c *chk.C) { + status := common.EJobStatus + + status = status.EnhanceJobStatusInfo(true, true, true) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithErrorsAndSkipped()) + + status = status.EnhanceJobStatusInfo(true, true, false) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithErrorsAndSkipped()) + + status = status.EnhanceJobStatusInfo(true, false, true) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithSkipped()) + + status = status.EnhanceJobStatusInfo(true, false, false) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithSkipped()) + + status = status.EnhanceJobStatusInfo(false, true, true) + c.Assert(status, chk.Equals, common.EJobStatus.CompletedWithErrors()) + + status = status.EnhanceJobStatusInfo(false, true, false) + c.Assert(status, chk.Equals, common.EJobStatus.Failed()) + + status = status.EnhanceJobStatusInfo(false, false, true) + c.Assert(status, chk.Equals, common.EJobStatus.Completed()) + + // No-op if all are false + status = status.EnhanceJobStatusInfo(false, false, false) + c.Assert(status, chk.Equals, common.EJobStatus.Completed()) +} + +func (s *feSteModelsTestSuite) TestIsJobDone(c *chk.C) { + status := common.EJobStatus.InProgress() + c.Assert(status.IsJobDone(), chk.Equals, false) + + status = status.Paused() + c.Assert(status.IsJobDone(), chk.Equals, false) + + status = status.Cancelling() + c.Assert(status.IsJobDone(), chk.Equals, false) + + status = status.Cancelled() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.Completed() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithErrors() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithSkipped() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithErrors() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.CompletedWithErrorsAndSkipped() + c.Assert(status.IsJobDone(), chk.Equals, true) + + status = status.Failed() + c.Assert(status.IsJobDone(), chk.Equals, true) +} \ No newline at end of file diff --git a/common/zt_credCache_test.go b/common/zt_credCache_test.go index 8d776c721..4fe314b18 100644 --- a/common/zt_credCache_test.go +++ b/common/zt_credCache_test.go @@ -49,7 +49,12 @@ var fakeTokenInfo = OAuthTokenInfo{ } func (s *credCacheTestSuite) TestCredCacheSaveLoadDeleteHas(c *chk.C) { - credCache := NewCredCache(".") // "." state is reserved to be used in Linux and MacOS, and used as path to save token file in Windows. + credCache := NewCredCache(CredCacheOptions{ + DPAPIFilePath: "", + KeyName: "", + ServiceName: "", + AccountName: "", + }) // "." state is reserved to be used in Linux and MacOS, and used as path to save token file in Windows. defer func() { // Cleanup fake token diff --git a/go.mod b/go.mod index 79de31b40..503c255c6 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,13 @@ require ( github.com/Azure/go-autorest v10.15.2+incompatible github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda github.com/danieljoos/wincred v1.0.1 - github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/inconshreveable/mousetrap v1.0.0 + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jiacfan/keychain v0.0.0-20180920053336-f2c902a3d807 github.com/jiacfan/keyctl v0.0.0-20160328205232-988d05162bc5 - github.com/kr/pretty v0.1.0 - github.com/kr/text v0.1.0 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.2 + github.com/stretchr/testify v1.3.0 // indirect golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 ) diff --git a/go.sum b/go.sum index 7f80d2b31..18919151e 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,7 @@ -github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876 h1:3c3mGlhASTJh6H6Ba9EHv2FDSmEUyJuJHR6UD7b+YuE= -github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.1.8 h1:KmVRa8oFMaargVesEuuEoiLCQ4zCCwQ8QX/xg++KS20= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-storage-blob-go v0.0.0-20180727221336-197d1c0aea1b h1:7cOe9XtL/0qFd/jb0whfm5NoRmhEcVU3bZ65zUZUz54= -github.com/Azure/azure-storage-blob-go v0.0.0-20180727221336-197d1c0aea1b/go.mod h1:x2mtS6O3mnMEZOJp7d7oldh8IvatBrMfReiyQ+cKgKY= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c h1:Y5ueznoCekgCWBytF1Q9lTpZ3tJeX37dQtCcGjMCLYI= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= -github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc h1:BElWmFfsryQD72OcovStKpkIcd4e9ozSkdsTNQDSHGk= -github.com/Azure/azure-storage-blob-go v0.0.0-20181023070848-cf01652132cc/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-file-go v0.0.0-20180929015327-d6c64f9676be h1:2TBD/QJYxkQf2jAHk/radyLgEfUpV8eqvRPvnjJt9EA= github.com/Azure/azure-storage-file-go v0.0.0-20180929015327-d6c64f9676be/go.mod h1:N5mXnKL8ZzcrxcNfqrcfWhiaCPAGagfTxH0/IwPN/LI= github.com/Azure/azure-storage-file-go v0.0.0-20190108093629-d93e19c84c2a h1:5OfEqciJHSMkxAWgJP1b3JTmzWNRlq9L9IgOxPNlBOM= @@ -18,6 +12,8 @@ github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNP github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= github.com/danieljoos/wincred v1.0.1 h1:fcRTaj17zzROVqni2FiToKUVg3MmJ4NtMSGCySPIr/g= github.com/danieljoos/wincred v1.0.1/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -31,10 +27,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190107173414-20be8e55dc7b h1:9Gu1sMPgKHo+qCbPa2jN5A54ro2gY99BWF7nHOBNVME= diff --git a/ste/init.go b/ste/init.go index ddb195ae9..1aececbb3 100644 --- a/ste/init.go +++ b/ste/init.go @@ -472,6 +472,12 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse { if (js.CompleteJobOrdered) && (part0PlanStatus == common.EJobStatus.Completed()) { js.JobStatus = part0PlanStatus } + + if js.JobStatus == common.EJobStatus.Completed() { + js.JobStatus = js.JobStatus.EnhanceJobStatusInfo(js.TransfersSkipped > 0, js.TransfersFailed > 0, + js.TransfersCompleted > 0) + } + return js } @@ -591,6 +597,13 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { if (js.CompleteJobOrdered) && (part0PlanStatus == common.EJobStatus.Completed()) { js.JobStatus = part0PlanStatus } + + if js.JobStatus == common.EJobStatus.Completed() { + js.JobStatus = js.JobStatus.EnhanceJobStatusInfo(false, + js.CopyTransfersFailed + js.DeleteTransfersFailed > 0, + js.CopyTransfersCompleted + js.DeleteTransfersCompleted > 0) + } + return js } From 43ca259e1df43011a5f83eaa06e1d013f492992f Mon Sep 17 00:00:00 2001 From: Ze Qian Zhang Date: Mon, 18 Feb 2019 19:24:58 -0800 Subject: [PATCH 09/64] Rework multiSliceBytePooler bucket allocation to be efficient when requested sizes are powers of two (#216) Because that's what they usually will be, in our usage --- common/multiSizeSlicePool.go | 35 ++++++++++---- common/zt_multiSliceBytePooler_test.go | 66 ++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 common/zt_multiSliceBytePooler_test.go diff --git a/common/multiSizeSlicePool.go b/common/multiSizeSlicePool.go index 5bee56fd4..cfb687d1e 100644 --- a/common/multiSizeSlicePool.go +++ b/common/multiSizeSlicePool.go @@ -92,17 +92,32 @@ func NewMultiSizeSlicePool(maxSliceLength uint32) ByteSlicePooler { } func getSlotInfo(exactSliceLength uint32) (slotIndex int, maxCapInSlot int) { - // slot index is fast computation of the base-2 logarithm, rounded down - slotIndex = 32 - bits.LeadingZeros32(exactSliceLength) - // max cap in slot is the biggest number that maps to that slot index - // (e.g. slot index of 1 (which=2 to the power of 0) is 1, so (2 to the power of slotIndex) - // is the first number that doesn't fit the slot) - maxCapInSlot = (1 << uint(slotIndex)) - 1 - - // check TODO: replace this check with a proper unit test - if 32-bits.LeadingZeros32(uint32(maxCapInSlot)) != slotIndex { - panic("cross check of cap and slot index failed") + if exactSliceLength <= 0 { + panic("exact slice length must be greater than zero") } + // raw slot index is fast computation of the base-2 logarithm, rounded down... + rawSlotIndex := 31 - bits.LeadingZeros32(exactSliceLength) + + // ...but in most cases we actually want to round up. + // E.g. we want 255 to go into the same bucket as 256. Why? because we want exact + // powers of 2 to be the largest thing in each bucket, since usually + // we will be using powers of 2, and that means we will usually be using + // all the allocated capacity (i.e. len == cap). That gives the most efficient use of RAM. + // The only time we don't want to round up, is if we already had an exact power of + // 2 to start with. + isExactPowerOfTwo := bits.OnesCount32(exactSliceLength) == 1 + if isExactPowerOfTwo { + slotIndex = rawSlotIndex + } else { + slotIndex = rawSlotIndex + 1 + } + + // Max cap in slot is the biggest number that maps to that slot index + // (e.g. slot index of exactSliceLength=1 (which=2 to the power of 0) + // is 0 (because log-base2 of 1 == 0), so (2 to the power of slotIndex) + // is the highest number that still fits the slot) + maxCapInSlot = 1 << uint(slotIndex) + return } diff --git a/common/zt_multiSliceBytePooler_test.go b/common/zt_multiSliceBytePooler_test.go new file mode 100644 index 000000000..817bc1667 --- /dev/null +++ b/common/zt_multiSliceBytePooler_test.go @@ -0,0 +1,66 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package common + +import ( + chk "gopkg.in/check.v1" + "math" +) + +type multiSliceBytePoolerSuite struct{} + +var _ = chk.Suite(&multiSliceBytePoolerSuite{}) + +func (s *multiSliceBytePoolerSuite) TestMultiSliceSlotInfo(c *chk.C) { + eightMB := 8 * 1024 * 1024 + + cases := []struct { + size int + expectedSlotIndex int + expectedMaxCapInSlot int + }{ + {1, 0, 1}, + {2, 1, 2}, + {3, 2, 4}, + {4, 2, 4}, + {5, 3, 8}, + {8, 3, 8}, + {9, 4, 16}, + {eightMB - 1, 23, eightMB}, + {eightMB, 23, eightMB}, + {eightMB + 1, 24, eightMB * 2}, + {100 * 1024 * 1024, 27, 128 * 1024 * 1024}, + } + + for _, x := range cases { + logBase2 := math.Log2(float64(x.size)) + roundedLogBase2 := int(math.Round(logBase2 + 0.49999999999999)) // rounds up unless already exact(ish) + + // now lets see if the pooler is working as we expect + slotIndex, maxCap := getSlotInfo(uint32(x.size)) + + c.Assert(slotIndex, chk.Equals, roundedLogBase2) // this what, mathematically, we expect + c.Assert(slotIndex, chk.Equals, x.expectedSlotIndex) // this what our test case said (should be same) + + c.Assert(maxCap, chk.Equals, x.expectedMaxCapInSlot) + } + +} From 7569d02ebc4fb89cef1f13aabf518472ea636728 Mon Sep 17 00:00:00 2001 From: Ze Qian Zhang Date: Mon, 18 Feb 2019 21:33:20 -0800 Subject: [PATCH 10/64] Fix/command line parameter naming (#217) * Convert remaining camelCase parameter names to be dash-separated * Fix capitalization of remaining inconsistent parameters * Add change log, and populate it with the parameter name changes --- ChangeLog.md | 11 +++++++++++ cmd/copy.go | 10 +++++----- cmd/credentialUtil.go | 2 +- cmd/list.go | 2 +- ste/mgr-JobPartMgr.go | 2 +- ste/xfer.go | 2 +- testSuite/scripts/test_upload_page_blob.py | 20 ++++++++++---------- testSuite/scripts/utility.py | 2 +- 8 files changed, 31 insertions(+), 20 deletions(-) create mode 100644 ChangeLog.md diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 000000000..0329bbfa4 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,11 @@ +# Change Log + +## Version X.XX.XX: + +- Command line parameter names changed as follows (to be consistent with naming pattern of other parameters) + - fromTo -> from-to + - blobType -> blob-type + - excludedBlobType -> excluded-blob-type + - outputRaw (in "list" command) -> output + - stdIn-enable (reserved for internal use) -> stdin-enable + diff --git a/cmd/copy.go b/cmd/copy.go index 60ca5c362..5d75a49cf 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -308,7 +308,7 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { var eBlobType common.BlobType err := eBlobType.Parse(blobType) if err != nil { - return cooked, fmt.Errorf("error parsing the excludeBlobType %s provided with excludeBlobTypeFlag ", blobType) + return cooked, fmt.Errorf("error parsing the exclude-blob-type %s provided with exclude-blob-type flag ", blobType) } cooked.excludeBlobType = append(cooked.excludeBlobType, eBlobType.ToAzBlobType()) } @@ -912,14 +912,14 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude these files when copying. Support use of *.") cpCmd.PersistentFlags().BoolVar(&raw.forceWrite, "overwrite", true, "overwrite the conflicting files/blobs at the destination if this flag is set to true.") cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when uploading from local file system.") - cpCmd.PersistentFlags().StringVar(&raw.fromTo, "fromTo", "", "optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.") - cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "excludeBlobType", "", "optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from Container / Account. Use of "+ + cpCmd.PersistentFlags().StringVar(&raw.fromTo, "from-to", "", "optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.") + cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "exclude-blob-type", "", "optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from Container / Account. Use of "+ "this flag is not applicable for copying data from non azure-service to service. More than one blob should be separated by ';' ") // options change how the transfers are performed cpCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json.") cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") cpCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "use this block(chunk) size when uploading/downloading to/from Azure Storage.") - cpCmd.PersistentFlags().StringVar(&raw.blobType, "blobType", "None", "defines the type of blob at the destination. This is used in case of upload / account to account copy") + cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "None", "defines the type of blob at the destination. This is used in case of upload / account to account copy") cpCmd.PersistentFlags().StringVar(&raw.blockBlobTier, "block-blob-tier", "None", "upload block blob to Azure Storage using this blob tier.") cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "upload page blob to Azure Storage using this blob tier.") cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "upload to Azure Storage with these key-value pairs as metadata.") @@ -943,7 +943,7 @@ func init() { cpCmd.PersistentFlags().MarkHidden("list-of-files") cpCmd.PersistentFlags().MarkHidden("include") cpCmd.PersistentFlags().MarkHidden("output") - cpCmd.PersistentFlags().MarkHidden("stdIn-enable") + cpCmd.PersistentFlags().MarkHidden("stdin-enable") cpCmd.PersistentFlags().MarkHidden("background-op") cpCmd.PersistentFlags().MarkHidden("cancel-from-stdin") } diff --git a/cmd/credentialUtil.go b/cmd/credentialUtil.go index d87f13022..ca0d29d0b 100644 --- a/cmd/credentialUtil.go +++ b/cmd/credentialUtil.go @@ -246,7 +246,7 @@ func getCredentialType(ctx context.Context, raw rawFromToInfo) (credentialType c default: credentialType = common.ECredentialType.Anonymous() // Log the FromTo types which getCredentialType hasn't solved, in case of miss-use. - glcm.Info(fmt.Sprintf("Use anonymous credential by default for FromTo '%v'", raw.fromTo)) + glcm.Info(fmt.Sprintf("Use anonymous credential by default for from-to '%v'", raw.fromTo)) } return credentialType, nil diff --git a/cmd/list.go b/cmd/list.go index f6cf97aa9..375b1b26a 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -76,7 +76,7 @@ func init() { }, } rootCmd.AddCommand(listContainerCmd) - listContainerCmd.PersistentFlags().StringVar(&outputRaw, "outputRaw", "text", "format of the command's outputRaw, the choices include: text, json") + listContainerCmd.PersistentFlags().StringVar(&outputRaw, "output", "text", "format of the command's output, the choices include: text, json") } // HandleListContainerCommand handles the list container command diff --git a/ste/mgr-JobPartMgr.go b/ste/mgr-JobPartMgr.go index 976e7be15..ea219b8a3 100644 --- a/ste/mgr-JobPartMgr.go +++ b/ste/mgr-JobPartMgr.go @@ -421,7 +421,7 @@ func (jpm *jobPartMgr) createPipeline(ctx context.Context) { }, jpm.pacer) default: - panic(fmt.Errorf("Unrecognized FromTo: %q", fromTo.String())) + panic(fmt.Errorf("Unrecognized from-to: %q", fromTo.String())) } } } diff --git a/ste/xfer.go b/ste/xfer.go index 6acad2cad..50c17c81e 100644 --- a/ste/xfer.go +++ b/ste/xfer.go @@ -96,7 +96,7 @@ func computeJobXfer(fromTo common.FromTo, blobType common.BlobType) newJobXfer { case common.EFromTo.FileBlob(): return URLToBlob } - panic(fmt.Errorf("Unrecognized FromTo: %q", fromTo.String())) + panic(fmt.Errorf("Unrecognized from-to: %q", fromTo.String())) } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/testSuite/scripts/test_upload_page_blob.py b/testSuite/scripts/test_upload_page_blob.py index 6bc723707..dfb465c42 100644 --- a/testSuite/scripts/test_upload_page_blob.py +++ b/testSuite/scripts/test_upload_page_blob.py @@ -18,7 +18,7 @@ def util_test_page_blob_upload_1mb(self, use_oauth=False): dest_validate = util.get_resource_from_oauth_container_validate(file_name) result = util.Command("copy").add_arguments(file_path).add_arguments(dest).add_flags("log-level", "info"). \ - add_flags("block-size", "4194304").add_flags("blobType","PageBlob").execute_azcopy_copy_command() + add_flags("block-size", "4194304").add_flags("blob-type","PageBlob").execute_azcopy_copy_command() self.assertTrue(result) # execute validator. @@ -46,7 +46,7 @@ def test_page_range_for_complete_sparse_file(self): # execute azcopy page blob upload. destination_sas = util.get_resource_sas(file_name) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \ - add_flags("block-size", "4194304").add_flags("blobType","PageBlob").execute_azcopy_copy_command() + add_flags("block-size", "4194304").add_flags("blob-type","PageBlob").execute_azcopy_copy_command() self.assertTrue(result) # execute validator. @@ -66,7 +66,7 @@ def test_page_blob_upload_partial_sparse_file(self): # execute azcopy pageblob upload. destination_sas = util.get_resource_sas(file_name) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("log-level", "info"). \ - add_flags("block-size", "4194304").add_flags("blobType","PageBlob").execute_azcopy_copy_command() + add_flags("block-size", "4194304").add_flags("blob-type","PageBlob").execute_azcopy_copy_command() self.assertTrue(result) # number of page range for partial sparse created above will be (size/2) @@ -85,7 +85,7 @@ def test_set_page_blob_tier(self): destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P10").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P10").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -99,7 +99,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P20").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P20").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -113,7 +113,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P30").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P30").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -127,7 +127,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P4").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P4").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -141,7 +141,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P40").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P40").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -155,7 +155,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P50").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P50").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. @@ -169,7 +169,7 @@ def test_set_page_blob_tier(self): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blobType","PageBlob").add_flags("page-blob-tier", "P6").execute_azcopy_copy_command() + add_flags("log-level", "info").add_flags("blob-type","PageBlob").add_flags("page-blob-tier", "P6").execute_azcopy_copy_command() self.assertTrue(result) # execute azcopy validate order. diff --git a/testSuite/scripts/utility.py b/testSuite/scripts/utility.py index 4bb77c3d0..55fb05b4d 100644 --- a/testSuite/scripts/utility.py +++ b/testSuite/scripts/utility.py @@ -84,7 +84,7 @@ def process_oauth_command( cmd, fromTo=""): if fromTo!="": - cmd.add_flags("fromTo", fromTo) + cmd.add_flags("from-to", fromTo) # api executes the clean command on validator which deletes all the contents of the container. def clean_test_container(container): From dadca8ce05154d99e88e037efa7733a182ff5f70 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 22 Jan 2019 17:08:21 -0800 Subject: [PATCH 11/64] Added integration testing PoC --- cmd/copyUtil_test.go | 7 +- cmd/zt_sync_download_test.go | 100 ++++++++++++++ cmd/zt_test.go | 256 +++++++++++++++++++++++++++++++++++ cmd/zt_test_interceptor.go | 66 +++++++++ 4 files changed, 423 insertions(+), 6 deletions(-) create mode 100644 cmd/zt_sync_download_test.go create mode 100644 cmd/zt_test.go create mode 100644 cmd/zt_test_interceptor.go diff --git a/cmd/copyUtil_test.go b/cmd/copyUtil_test.go index f5ffbfa79..879feaeac 100644 --- a/cmd/copyUtil_test.go +++ b/cmd/copyUtil_test.go @@ -21,15 +21,10 @@ package cmd import ( - "net/url" - "testing" - chk "gopkg.in/check.v1" + "net/url" ) -// Hookup to the testing framework -func Test(t *testing.T) { chk.TestingT(t) } - type copyUtilTestSuite struct{} var _ = chk.Suite(©UtilTestSuite{}) diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go new file mode 100644 index 000000000..5823b91ab --- /dev/null +++ b/cmd/zt_sync_download_test.go @@ -0,0 +1,100 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "io/ioutil" + "strings" +) + +// create a test file +func generateFile(fileName string, fileSize int) ([]byte, error) { + // generate random data + _, bigBuff := getRandomDataAndReader(fileSize) + + // write to file and return the data + err := ioutil.WriteFile(fileName, bigBuff, 0666) + return bigBuff, err +} + +// create the necessary blobs with and without virtual directories +func generateCommonScenarioForDownloadSync(c *chk.C) (containerName string, containerUrl azblob.ContainerURL, blobList []string) { + bsu := getBSU() + containerUrl, containerName = createNewContainer(c, bsu) + + blobList = make([]string, 30) + for i := 0; i < 10; i++ { + _, blobName1 := createNewBlockBlob(c, containerUrl, "top") + _, blobName2 := createNewBlockBlob(c, containerUrl, "sub1/") + _, blobName3 := createNewBlockBlob(c, containerUrl, "sub2/") + + blobList[3*i] = blobName1 + blobList[3*i+1] = blobName2 + blobList[3*i+2] = blobName3 + } + + return +} + +// Golang does not have sets, so we have to use a map to fulfill the same functionality +func convertListToMap(list []string) map[string]int { + lookupMap := make(map[string]int) + for _, entryName := range list { + lookupMap[entryName] = 0 + } + + return lookupMap +} + +func (s *cmdIntegrationSuite) TestSyncDownloadWithEmptyDestination(c *chk.C) { + // set up the container with numerous blobs + containerName, containerURL, blobList := generateCommonScenarioForDownloadSync(c) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination with an empty folder + dstDirName, err := ioutil.TempDir("", "AzCopySyncDownload") + c.Assert(err, chk.IsNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + credential, err := getGenericCredential("") + c.Assert(err, chk.IsNil) + containerUrlWithSAS := getContainerURLWithSAS(c, *credential, containerName) + rawContainerURLWithSAS := containerUrlWithSAS.URL() + raw := rawSyncCmdArgs{ + src: rawContainerURLWithSAS.String(), + dst: dstDirName, + recursive: true, + logVerbosity: "WARNING", + output: "text", + force: false, + } + + // the simulated user input should parse properly + cooked, err := raw.cook() + c.Assert(err, chk.IsNil) + + // the enumeration ends when process() returns + err = cooked.process() + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 30) + + // validate that the right transfers were sent + lookupMap := convertListToMap(blobList) + for _, transfer := range mockedRPC.transfers { + localRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + + // look up the source blob, make sure it matches + _, blobExist := lookupMap[localRelativeFilePath] + c.Assert(blobExist, chk.Equals, true) + } +} diff --git a/cmd/zt_test.go b/cmd/zt_test.go new file mode 100644 index 000000000..10253880c --- /dev/null +++ b/cmd/zt_test.go @@ -0,0 +1,256 @@ +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/ioutil" + "net/url" + "os" + "runtime" + "strings" + "testing" + "time" + + chk "gopkg.in/check.v1" + + "math/rand" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +// Hookup to the testing framework +func Test(t *testing.T) { chk.TestingT(t) } + +type cmdIntegrationSuite struct{} + +var _ = chk.Suite(&cmdIntegrationSuite{}) +var ctx = context.Background() + +const ( + containerPrefix = "container" + blobPrefix = "blob" + blockBlobDefaultData = "AzCopy Random Test Data" +) + +// This function generates an entity name by concatenating the passed prefix, +// the name of the test requesting the entity name, and the minute, second, and nanoseconds of the call. +// This should make it easy to associate the entities with their test, uniquely identify +// them, and determine the order in which they were created. +// Note that this imposes a restriction on the length of test names +func generateName(prefix string) string { + // These next lines up through the for loop are obtaining and walking up the stack + // trace to extrat the test name, which is stored in name + pc := make([]uintptr, 10) + runtime.Callers(0, pc) + f := runtime.FuncForPC(pc[0]) + name := f.Name() + for i := 0; !strings.Contains(name, "Suite"); i++ { // The tests are all scoped to the suite, so this ensures getting the actual test name + f = runtime.FuncForPC(pc[i]) + name = f.Name() + } + funcNameStart := strings.Index(name, "Test") + name = name[funcNameStart+len("Test"):] // Just get the name of the test and not any of the garbage at the beginning + name = strings.ToLower(name) // Ensure it is a valid resource name + currentTime := time.Now() + name = fmt.Sprintf("%s%s%d%d%d", prefix, strings.ToLower(name), currentTime.Minute(), currentTime.Second(), currentTime.Nanosecond()) + return name +} + +func generateContainerName() string { + return generateName(containerPrefix) +} + +func generateBlobName() string { + return generateName(blobPrefix) +} + +func getContainerURL(c *chk.C, bsu azblob.ServiceURL) (container azblob.ContainerURL, name string) { + name = generateContainerName() + container = bsu.NewContainerURL(name) + + return container, name +} + +func getBlockBlobURL(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.BlockBlobURL, name string) { + name = prefix + generateBlobName() + blob = container.NewBlockBlobURL(name) + + return blob, name +} + +func getAppendBlobURL(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.AppendBlobURL, name string) { + name = generateBlobName() + blob = container.NewAppendBlobURL(prefix + name) + + return blob, name +} + +func getPageBlobURL(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.PageBlobURL, name string) { + name = generateBlobName() + blob = container.NewPageBlobURL(prefix + name) + + return +} + +func getReaderToRandomBytes(n int) *bytes.Reader { + r, _ := getRandomDataAndReader(n) + return r +} + +func getRandomDataAndReader(n int) (*bytes.Reader, []byte) { + data := make([]byte, n, n) + rand.Read(data) + return bytes.NewReader(data), data +} + +func createNewContainer(c *chk.C, bsu azblob.ServiceURL) (container azblob.ContainerURL, name string) { + container, name = getContainerURL(c, bsu) + + cResp, err := container.Create(ctx, nil, azblob.PublicAccessNone) + c.Assert(err, chk.IsNil) + c.Assert(cResp.StatusCode(), chk.Equals, 201) + return container, name +} + +func createNewBlockBlob(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.BlockBlobURL, name string) { + blob, name = getBlockBlobURL(c, container, prefix) + + cResp, err := blob.Upload(ctx, strings.NewReader(blockBlobDefaultData), azblob.BlobHTTPHeaders{}, + nil, azblob.BlobAccessConditions{}) + + c.Assert(err, chk.IsNil) + c.Assert(cResp.StatusCode(), chk.Equals, 201) + + return +} + +func createNewAppendBlob(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.AppendBlobURL, name string) { + blob, name = getAppendBlobURL(c, container, prefix) + + resp, err := blob.Create(ctx, azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{}) + + c.Assert(err, chk.IsNil) + c.Assert(resp.StatusCode(), chk.Equals, 201) + return +} + +func createNewPageBlob(c *chk.C, container azblob.ContainerURL, prefix string) (blob azblob.PageBlobURL, name string) { + blob, name = getPageBlobURL(c, container, prefix) + + resp, err := blob.Create(ctx, azblob.PageBlobPageBytes*10, 0, azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{}) + + c.Assert(err, chk.IsNil) + c.Assert(resp.StatusCode(), chk.Equals, 201) + return +} + +func deleteContainer(c *chk.C, container azblob.ContainerURL) { + resp, err := container.Delete(ctx, azblob.ContainerAccessConditions{}) + c.Assert(err, chk.IsNil) + c.Assert(resp.StatusCode(), chk.Equals, 202) +} + +func getGenericCredential(accountType string) (*azblob.SharedKeyCredential, error) { + accountNameEnvVar := accountType + "ACCOUNT_NAME" + accountKeyEnvVar := accountType + "ACCOUNT_KEY" + accountName, accountKey := os.Getenv(accountNameEnvVar), os.Getenv(accountKeyEnvVar) + if accountName == "" || accountKey == "" { + return nil, errors.New(accountNameEnvVar + " and/or " + accountKeyEnvVar + " environment variables not specified.") + } + return azblob.NewSharedKeyCredential(accountName, accountKey) +} + +func getGenericBSU(accountType string) (azblob.ServiceURL, error) { + credential, err := getGenericCredential(accountType) + if err != nil { + return azblob.ServiceURL{}, err + } + + pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + blobPrimaryURL, _ := url.Parse("https://" + credential.AccountName() + ".blob.core.windows.net/") + return azblob.NewServiceURL(*blobPrimaryURL, pipeline), nil +} + +func getBSU() azblob.ServiceURL { + bsu, _ := getGenericBSU("") + return bsu +} + +func validateStorageError(c *chk.C, err error, code azblob.ServiceCodeType) { + serr, _ := err.(azblob.StorageError) + c.Assert(serr.ServiceCode(), chk.Equals, code) +} + +func getRelativeTimeGMT(amount time.Duration) time.Time { + currentTime := time.Now().In(time.FixedZone("GMT", 0)) + currentTime = currentTime.Add(amount * time.Second) + return currentTime +} + +func generateCurrentTimeWithModerateResolution() time.Time { + highResolutionTime := time.Now().UTC() + return time.Date(highResolutionTime.Year(), highResolutionTime.Month(), highResolutionTime.Day(), highResolutionTime.Hour(), highResolutionTime.Minute(), + highResolutionTime.Second(), 0, highResolutionTime.Location()) +} + +// Some tests require setting service properties. It can take up to 30 seconds for the new properties to be reflected across all FEs. +// We will enable the necessary property and try to run the test implementation. If it fails with an error that should be due to +// those changes not being reflected yet, we will wait 30 seconds and try the test again. If it fails this time for any reason, +// we fail the test. It is the responsibility of the the testImplFunc to determine which error string indicates the test should be retried. +// There can only be one such string. All errors that cannot be due to this detail should be asserted and not returned as an error string. +func runTestRequiringServiceProperties(c *chk.C, bsu azblob.ServiceURL, code string, + enableServicePropertyFunc func(*chk.C, azblob.ServiceURL), + testImplFunc func(*chk.C, azblob.ServiceURL) error, + disableServicePropertyFunc func(*chk.C, azblob.ServiceURL)) { + enableServicePropertyFunc(c, bsu) + defer disableServicePropertyFunc(c, bsu) + err := testImplFunc(c, bsu) + // We cannot assume that the error indicative of slow update will necessarily be a StorageError. As in ListBlobs. + if err != nil && err.Error() == code { + time.Sleep(time.Second * 30) + err = testImplFunc(c, bsu) + c.Assert(err, chk.IsNil) + } +} + +func enableSoftDelete(c *chk.C, bsu azblob.ServiceURL) { + days := int32(1) + _, err := bsu.SetProperties(ctx, azblob.StorageServiceProperties{DeleteRetentionPolicy: &azblob.RetentionPolicy{Enabled: true, Days: &days}}) + c.Assert(err, chk.IsNil) +} + +func disableSoftDelete(c *chk.C, bsu azblob.ServiceURL) { + _, err := bsu.SetProperties(ctx, azblob.StorageServiceProperties{DeleteRetentionPolicy: &azblob.RetentionPolicy{Enabled: false}}) + c.Assert(err, chk.IsNil) +} + +func validateUpload(c *chk.C, blobURL azblob.BlockBlobURL) { + resp, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false) + c.Assert(err, chk.IsNil) + data, _ := ioutil.ReadAll(resp.Response().Body) + c.Assert(data, chk.HasLen, 0) +} + +func getContainerURLWithSAS(c *chk.C, credential azblob.SharedKeyCredential, containerName string) azblob.ContainerURL { + sasQueryParams, err := azblob.BlobSASSignatureValues{ + Protocol: azblob.SASProtocolHTTPS, + ExpiryTime: time.Now().UTC().Add(48 * time.Hour), + ContainerName: containerName, + Permissions: azblob.ContainerSASPermissions{Read: true, Add: true, Write: true, Create: true, Delete: true, List: true}.String(), + }.NewSASQueryParameters(&credential) + c.Assert(err, chk.IsNil) + + // construct the url from scratch + qp := sasQueryParams.Encode() + rawURL := fmt.Sprintf("https://%s.blob.core.windows.net/%s?%s", + credential.AccountName(), containerName, qp) + + // convert the raw url and validate it was parsed successfully + fullURL, err := url.Parse(rawURL) + c.Assert(err, chk.IsNil) + + // TODO perhaps we need a global default pipeline + return azblob.NewContainerURL(*fullURL, azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{})) +} diff --git a/cmd/zt_test_interceptor.go b/cmd/zt_test_interceptor.go new file mode 100644 index 000000000..f254cf55a --- /dev/null +++ b/cmd/zt_test_interceptor.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + "time" +) + +// the interceptor gathers/saves the job part orders for validation +type interceptor struct { + transfers []common.CopyTransfer + lastRequest interface{} +} + +func (i *interceptor) intercept(cmd common.RpcCmd, request interface{}, response interface{}) { + switch cmd { + case common.ERpcCmd.CopyJobPartOrder(): + // cache the transfers + copyRequest := *request.(*common.CopyJobPartOrderRequest) + i.transfers = append(i.transfers, copyRequest.Transfers...) + i.lastRequest = request + + // mock the result + *(response.(*common.CopyJobPartOrderResponse)) = common.CopyJobPartOrderResponse{JobStarted: true} + + case common.ERpcCmd.ListSyncJobSummary(): + copyRequest := *request.(*common.CopyJobPartOrderRequest) + + // fake the result saying that job is already completed + // doing so relies on the mockedLifecycleManager not quitting the application + *(response.(*common.ListSyncJobSummaryResponse)) = common.ListSyncJobSummaryResponse{ + Timestamp: time.Now().UTC(), + JobID: copyRequest.JobID, + ErrorMsg: "", + JobStatus: common.EJobStatus.Completed(), + CompleteJobOrdered: true, + FailedTransfers: []common.TransferDetail{}, + } + case common.ERpcCmd.ListJobs(): + case common.ERpcCmd.ListJobSummary(): + case common.ERpcCmd.ListJobTransfers(): + case common.ERpcCmd.PauseJob(): + case common.ERpcCmd.CancelJob(): + case common.ERpcCmd.ResumeJob(): + case common.ERpcCmd.GetJobFromTo(): + fallthrough + default: + panic("RPC mock not implemented") + } +} + +func (i *interceptor) init() { + // mock out the lifecycle manager so that it can no longer terminate the application + glcm = mockedLifecycleManager{} +} + +// this lifecycle manager substitute does not perform any action +type mockedLifecycleManager struct{} + +func (mockedLifecycleManager) Progress(string) {} +func (mockedLifecycleManager) Info(string) {} +func (mockedLifecycleManager) Prompt(string) string { return "" } +func (mockedLifecycleManager) Exit(string, common.ExitCode) {} +func (mockedLifecycleManager) Error(string) {} +func (mockedLifecycleManager) SurrenderControl() {} +func (mockedLifecycleManager) InitiateProgressReporting(common.WorkController, bool) {} +func (mockedLifecycleManager) GetEnvironmentVariable(common.EnvironmentVariable) string { return "" } From d1d8bd75f2f70c0dccbc0ca6433ad773cb7f74a6 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 22 Jan 2019 17:11:20 -0800 Subject: [PATCH 12/64] Rewrote sync command + new architecture + added new integration tests --- cmd/copyEnumeratorHelper.go | 2 + cmd/copyUploadEnumerator.go | 6 +- cmd/copyUtil.go | 8 +- cmd/rpc.go | 1 - cmd/sync.go | 260 ++++----- cmd/syncDownloadEnumerator.go | 486 ----------------- cmd/syncEnumerator.go | 242 +++++++++ cmd/syncFilter.go | 71 +++ cmd/syncProcessor.go | 181 +++++++ cmd/syncTraverser.go | 268 ++++++++++ cmd/syncUploadEnumerator.go | 563 -------------------- cmd/zt_scenario_helpers_test.go | 110 ++++ cmd/zt_sync_download_test.go | 473 ++++++++++++++-- cmd/zt_sync_filter_test.go | 52 ++ cmd/zt_sync_processor_test.go | 131 +++++ cmd/zt_sync_traverser_test.go | 181 +++++++ cmd/zt_test_interceptor.go | 5 + common/rpc-models.go | 6 +- go.mod | 2 +- go.sum | 2 + testSuite/scripts/test_blob_download.py | 131 ----- testSuite/scripts/test_upload_block_blob.py | 185 ------- 22 files changed, 1772 insertions(+), 1594 deletions(-) delete mode 100644 cmd/syncDownloadEnumerator.go create mode 100644 cmd/syncEnumerator.go create mode 100644 cmd/syncFilter.go create mode 100644 cmd/syncProcessor.go create mode 100644 cmd/syncTraverser.go delete mode 100644 cmd/syncUploadEnumerator.go create mode 100644 cmd/zt_scenario_helpers_test.go create mode 100644 cmd/zt_sync_filter_test.go create mode 100644 cmd/zt_sync_processor_test.go create mode 100644 cmd/zt_sync_traverser_test.go diff --git a/cmd/copyEnumeratorHelper.go b/cmd/copyEnumeratorHelper.go index d1f93c21e..23ad2f23b 100644 --- a/cmd/copyEnumeratorHelper.go +++ b/cmd/copyEnumeratorHelper.go @@ -35,6 +35,8 @@ func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer e.PartNum++ } + // only append the transfer after we've checked and dispatched a part + // so that there is at least one transfer for the final part e.Transfers = append(e.Transfers, transfer) return nil diff --git a/cmd/copyUploadEnumerator.go b/cmd/copyUploadEnumerator.go index 8a438aa6b..a3fd88848 100644 --- a/cmd/copyUploadEnumerator.go +++ b/cmd/copyUploadEnumerator.go @@ -87,7 +87,7 @@ func (e *copyUploadEnumerator) enumerate(cca *cookedCopyCmdArgs) error { if len(cca.listOfFilesToCopy) > 0 { for _, file := range cca.listOfFilesToCopy { tempDestinationURl := *destinationURL - parentSourcePath, _ := util.sourceRootPathWithoutWildCards(cca.source) + parentSourcePath, _ := util.getRootPathWithoutWildCards(cca.source) if len(parentSourcePath) > 0 && parentSourcePath[len(parentSourcePath)-1] == common.AZCOPY_PATH_SEPARATOR_CHAR { parentSourcePath = parentSourcePath[:len(parentSourcePath)-1] } @@ -283,7 +283,7 @@ func (e *copyUploadEnumerator) enumerate(cca *cookedCopyCmdArgs) error { return err } } - }else{ + } else { glcm.Info(fmt.Sprintf("error %s accessing the filepath %s", err.Error(), fileOrDirectoryPath)) } } @@ -303,7 +303,7 @@ func (e *copyUploadEnumerator) enumerate(cca *cookedCopyCmdArgs) error { // ffile1, ffile2, then destination for ffile1, ffile2 remotely will be MountedD/MountedF/ffile1 and // MountedD/MountedF/ffile2 func (e *copyUploadEnumerator) getSymlinkTransferList(symlinkPath, source, parentSource, cleanContainerPath string, - destinationUrl *url.URL, cca *cookedCopyCmdArgs) error{ + destinationUrl *url.URL, cca *cookedCopyCmdArgs) error { util := copyHandlerUtil{} // replace the "\\" path separator with "/" separator diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index 3d04253ad..d68d9319c 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -119,9 +119,9 @@ func (util copyHandlerUtil) ConstructCommandStringFromArgs() string { } s := strings.Builder{} for _, arg := range args { - // If the argument starts with https, it is either the remote source or remote destination + // If the argument starts with http, it is either the remote source or remote destination // If there exists a signature in the argument string it needs to be redacted - if startsWith(arg, "https") { + if startsWith(arg, "http") { // parse the url argUrl, err := url.Parse(arg) // If there is an error parsing the url, then throw the error @@ -414,10 +414,10 @@ func (util copyHandlerUtil) appendBlobNameToUrl(blobUrlParts azblob.BlobURLParts return blobUrlParts.URL(), blobUrlParts.BlobName } -// sourceRootPathWithoutWildCards returns the directory from path that does not have wildCards +// getRootPathWithoutWildCards returns the directory from path that does not have wildCards // returns the patterns that defines pattern for relativePath of files to the above mentioned directory // For Example: source = C:\User\a*\a1*\*.txt rootDir = C:\User\ pattern = a*\a1*\*.txt -func (util copyHandlerUtil) sourceRootPathWithoutWildCards(path string) (string, string) { +func (util copyHandlerUtil) getRootPathWithoutWildCards(path string) (string, string) { if len(path) == 0 { return path, "*" } diff --git a/cmd/rpc.go b/cmd/rpc.go index 220a30d67..034fed275 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -14,7 +14,6 @@ import ( // Global singleton for sending RPC requests from the frontend to the STE var Rpc = func(cmd common.RpcCmd, request interface{}, response interface{}) { err := inprocSend(cmd, request, response) - //err := NewHttpClient("").send(cmd, request, response) common.PanicIfErr(err) } diff --git a/cmd/sync.go b/cmd/sync.go index 6de690040..de0e230bc 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -23,6 +23,7 @@ package cmd import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -35,10 +36,11 @@ import ( "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-azcopy/ste" "github.com/Azure/azure-storage-blob-go/azblob" - "github.com/Azure/azure-storage-file-go/azfile" "github.com/spf13/cobra" ) +// TODO +// a max is set because we cannot buffer infinite amount of destination file info in memory const MaxNumberOfFilesAllowedInSync = 10000000 type rawSyncCmdArgs struct { @@ -59,26 +61,77 @@ type rawSyncCmdArgs struct { force bool } +func (raw *rawSyncCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) { + cookedPatterns = make([]string, 0) + rawPatterns := strings.Split(pattern, ";") + for _, pattern := range rawPatterns { + + // skip the empty patterns + if len(pattern) != 0 { + cookedPatterns = append(cookedPatterns, pattern) + } + } + + return +} + +// given a valid URL, parse out the SAS portion +func (raw *rawSyncCmdArgs) separateSasFromURL(rawURL string) (cleanURL string, sas string) { + fromUrl, _ := url.Parse(rawURL) + + // TODO add support for other service URLs + blobParts := azblob.NewBlobURLParts(*fromUrl) + sas = blobParts.SAS.Encode() + + // get clean URL without SAS + blobParts.SAS = azblob.SASQueryParameters{} + bUrl := blobParts.URL() + cleanURL = bUrl.String() + + return +} + +func (raw *rawSyncCmdArgs) cleanLocalPath(rawPath string) (cleanPath string) { + // if the path separator is '\\', it means + // local path is a windows path + // to avoid path separator check and handling the windows + // path differently, replace the path separator with the + // the linux path separator '/' + if os.PathSeparator == '\\' { + cleanPath = strings.Replace(rawPath, common.OS_PATH_SEPARATOR, "/", -1) + } + cleanPath = rawPath + return +} + // validates and transform raw input into cooked input -func (raw rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { +func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { cooked := cookedSyncCmdArgs{} - fromTo := inferFromTo(raw.src, raw.dst) - if fromTo == common.EFromTo.Unknown() { + cooked.fromTo = inferFromTo(raw.src, raw.dst) + if cooked.fromTo == common.EFromTo.Unknown() { return cooked, fmt.Errorf("Unable to infer the source '%s' / destination '%s'. ", raw.src, raw.dst) + } else if cooked.fromTo == common.EFromTo.LocalBlob() { + cooked.source = raw.cleanLocalPath(raw.src) + cooked.destination, cooked.destinationSAS = raw.separateSasFromURL(raw.dst) + } else if cooked.fromTo == common.EFromTo.BlobLocal() { + cooked.source, cooked.sourceSAS = raw.separateSasFromURL(raw.src) + cooked.destination = raw.cleanLocalPath(raw.dst) + } else { + return cooked, fmt.Errorf("source '%s' / destination '%s' combination '%s' not supported for sync command ", raw.src, raw.dst, cooked.fromTo) } - if fromTo != common.EFromTo.LocalBlob() && - fromTo != common.EFromTo.BlobLocal() { - return cooked, fmt.Errorf("source '%s' / destination '%s' combination '%s' not supported for sync command ", raw.src, raw.dst, fromTo) - } - cooked.source = raw.src - cooked.destination = raw.dst - cooked.fromTo = fromTo + // generate a new job ID + cooked.jobID = common.NewJobID() cooked.blockSize = raw.blockSize - cooked.followSymlinks = raw.followSymlinks + cooked.recursive = raw.recursive + cooked.force = raw.force + + // parse the filter patterns + cooked.include = raw.parsePatterns(raw.include) + cooked.exclude = raw.parsePatterns(raw.exclude) err := cooked.logVerbosity.Parse(raw.logVerbosity) if err != nil { @@ -93,42 +146,11 @@ func (raw rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { return cooked, err } - // initialize the include map which contains the list of files to be included - // parse the string passed in include flag - // more than one file are expected to be separated by ';' - cooked.include = make(map[string]int) - if len(raw.include) > 0 { - files := strings.Split(raw.include, ";") - for index := range files { - // If split of the include string leads to an empty string - // not include that string - if len(files[index]) == 0 { - continue - } - cooked.include[files[index]] = index - } - } - - // initialize the exclude map which contains the list of files to be excluded - // parse the string passed in exclude flag - // more than one file are expected to be separated by ';' - cooked.exclude = make(map[string]int) - if len(raw.exclude) > 0 { - files := strings.Split(raw.exclude, ";") - for index := range files { - // If split of the include string leads to an empty string - // not include that string - if len(files[index]) == 0 { - continue - } - cooked.exclude[files[index]] = index - } + err = cooked.output.Parse(raw.output) + if err != nil { + return cooked, err } - cooked.recursive = raw.recursive - cooked.output.Parse(raw.output) - cooked.jobID = common.NewJobID() - cooked.force = raw.force return cooked, nil } @@ -138,15 +160,20 @@ type cookedSyncCmdArgs struct { destination string destinationSAS string fromTo common.FromTo + credentialInfo common.CredentialInfo + + // filters recursive bool followSymlinks bool - // options from flags - include map[string]int - exclude map[string]int + include []string + exclude []string + + // options + md5ValidationOption common.HashValidationOption blockSize uint32 logVerbosity common.LogLevel output common.OutputFormat - md5ValidationOption common.HashValidationOption + // commandString hold the user given command which is logged to the Job log file commandString string @@ -177,9 +204,9 @@ type cookedSyncCmdArgs struct { atomicSourceFilesScanned uint64 // defines the number of files listed at the destination and compared. atomicDestinationFilesScanned uint64 - // this flag predefines the user-agreement to delete the files in case sync found some files at destination - // which doesn't exists at source. With this flag turned on, user will not be asked for permission before - // deleting the flag. + // this flag determines the user-agreement to delete the files in case sync found files/blobs at the destination + // that do not exist at the source. With this flag turned on, user will not be prompted for permission before + // deleting the files/blobs. force bool } @@ -230,7 +257,7 @@ func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { // prompt for confirmation, except when: // 1. output is in json format // 2. enumeration is complete - if !(cca.output == common.EOutputFormat.Json() || cca.isEnumerationComplete) { + if cca.output == common.EOutputFormat.Text() && !cca.isEnumerationComplete { answer := lcm.Prompt("The source enumeration is not complete, cancelling the job at this point means it cannot be resumed. Please confirm with y/n: ") // read a line from stdin, if the answer is not yes, then abort cancel by returning @@ -246,7 +273,6 @@ func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { } func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { - if !cca.scanningComplete() { lcm.Progress(fmt.Sprintf("%v File Scanned at Source, %v Files Scanned at Destination", atomic.LoadUint64(&cca.atomicSourceFilesScanned), atomic.LoadUint64(&cca.atomicDestinationFilesScanned))) @@ -324,36 +350,25 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { } func (cca *cookedSyncCmdArgs) process() (err error) { - // initialize the fields that are constant across all job part orders - jobPartOrder := common.SyncJobPartOrderRequest{ - JobID: cca.jobID, - FromTo: cca.fromTo, - LogLevel: cca.logVerbosity, - BlockSizeInBytes: cca.blockSize, - Include: cca.include, - Exclude: cca.exclude, - CommandString: cca.commandString, - SourceSAS: cca.sourceSAS, - DestinationSAS: cca.destinationSAS, - CredentialInfo: common.CredentialInfo{}, - } - ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + // verifies credential type and initializes credential info. // For sync, only one side need credential. - if jobPartOrder.CredentialInfo.CredentialType, err = getCredentialType(ctx, rawFromToInfo{ + cca.credentialInfo.CredentialType, err = getCredentialType(ctx, rawFromToInfo{ fromTo: cca.fromTo, source: cca.source, destination: cca.destination, sourceSAS: cca.sourceSAS, destinationSAS: cca.destinationSAS, - }); err != nil { + }) + + if err != nil { return err } // For OAuthToken credential, assign OAuthTokenInfo to CopyJobPartOrderRequest properly, // the info will be transferred to STE. - if jobPartOrder.CredentialInfo.CredentialType == common.ECredentialType.OAuthToken() { + if cca.credentialInfo.CredentialType == common.ECredentialType.OAuthToken() { // Message user that they are using Oauth token for authentication, // in case of silently using cached token without consciousness。 glcm.Info("Using OAuth token for authentication.") @@ -363,94 +378,25 @@ func (cca *cookedSyncCmdArgs) process() (err error) { if tokenInfo, err := uotm.GetTokenInfo(ctx); err != nil { return err } else { - jobPartOrder.CredentialInfo.OAuthTokenInfo = *tokenInfo - } - } - - from := cca.fromTo.From() - to := cca.fromTo.To() - switch from { - case common.ELocation.Blob(): - fromUrl, err := url.Parse(cca.source) - if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", fromUrl.String(), err.Error()) + cca.credentialInfo.OAuthTokenInfo = *tokenInfo } - blobParts := azblob.NewBlobURLParts(*fromUrl) - cca.sourceSAS = blobParts.SAS.Encode() - jobPartOrder.SourceSAS = cca.sourceSAS - blobParts.SAS = azblob.SASQueryParameters{} - bUrl := blobParts.URL() - cca.source = bUrl.String() - case common.ELocation.File(): - fromUrl, err := url.Parse(cca.source) - if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", fromUrl.String(), err.Error()) - } - fileParts := azfile.NewFileURLParts(*fromUrl) - cca.sourceSAS = fileParts.SAS.Encode() - jobPartOrder.SourceSAS = cca.sourceSAS - fileParts.SAS = azfile.SASQueryParameters{} - fUrl := fileParts.URL() - cca.source = fUrl.String() } - switch to { - case common.ELocation.Blob(): - toUrl, err := url.Parse(cca.destination) - if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", toUrl.String(), err.Error()) - } - blobParts := azblob.NewBlobURLParts(*toUrl) - cca.destinationSAS = blobParts.SAS.Encode() - jobPartOrder.DestinationSAS = cca.destinationSAS - blobParts.SAS = azblob.SASQueryParameters{} - bUrl := blobParts.URL() - cca.destination = bUrl.String() - case common.ELocation.File(): - toUrl, err := url.Parse(cca.destination) - if err != nil { - return fmt.Errorf("error parsing the source url %s. Failed with error %s", toUrl.String(), err.Error()) - } - fileParts := azfile.NewFileURLParts(*toUrl) - cca.destinationSAS = fileParts.SAS.Encode() - jobPartOrder.DestinationSAS = cca.destinationSAS - fileParts.SAS = azfile.SASQueryParameters{} - fUrl := fileParts.URL() - cca.destination = fUrl.String() - } - - if from == common.ELocation.Local() { - // If the path separator is '\\', it means - // local path is a windows path - // To avoid path separator check and handling the windows - // path differently, replace the path separator with the - // the linux path separator '/' - if os.PathSeparator == '\\' { - cca.source = strings.Replace(cca.source, common.OS_PATH_SEPARATOR, "/", -1) - } - } - - if to == common.ELocation.Local() { - // If the path separator is '\\', it means - // local path is a windows path - // To avoid path separator check and handling the windows - // path differently, replace the path separator with the - // the linux path separator '/' - if os.PathSeparator == '\\' { - cca.destination = strings.Replace(cca.destination, common.OS_PATH_SEPARATOR, "/", -1) - } - } + var enumerator *syncEnumerator switch cca.fromTo { case common.EFromTo.LocalBlob(): - e := syncUploadEnumerator(jobPartOrder) - err = e.enumerate(cca) + return errors.New("work in progress") case common.EFromTo.BlobLocal(): - e := syncDownloadEnumerator(jobPartOrder) - err = e.enumerate(cca) + enumerator, err = newSyncDownloadEnumerator(cca) + if err != nil { + return err + } default: - return fmt.Errorf("from to destination not supported") + return fmt.Errorf("the given source/destination pair is currently not supported") } + + err = enumerator.enumerate() if err != nil { return fmt.Errorf("error starting the sync between source %s and destination %s. Failed with error %s", cca.source, cca.destination, err.Error()) } @@ -489,13 +435,11 @@ func init() { } rootCmd.AddCommand(syncCmd) - syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Filter: Look into sub-directories recursively when syncing destination to source.") - syncCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "Use this block size when source to Azure Storage or from Azure Storage.") - // hidden filters - syncCmd.PersistentFlags().StringVar(&raw.include, "include", "", "Filter: only include these files when copying. "+ - "Support use of *. More than one file are separated by ';'") - syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "Filter: Follow symbolic links when performing sync from local file system.") - syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "Filter: Exclude these files when copying. Support use of *.") + syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when syncing between directories.") + syncCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "use this block(chunk) size when uploading/downloading to/from Azure Storage.") + syncCmd.PersistentFlags().StringVar(&raw.include, "include", "", "only include files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") + syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") + syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "follow symbolic links when performing sync from local file system.") syncCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "WARNING", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") syncCmd.PersistentFlags().BoolVar(&raw.force, "force", false, "defines user's decision to delete extra files at the destination that are not present at the source. "+ diff --git a/cmd/syncDownloadEnumerator.go b/cmd/syncDownloadEnumerator.go deleted file mode 100644 index a4d62d6c3..000000000 --- a/cmd/syncDownloadEnumerator.go +++ /dev/null @@ -1,486 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - - "sync/atomic" - - "time" - - "github.com/Azure/azure-pipeline-go/pipeline" - "github.com/Azure/azure-storage-azcopy/common" - "github.com/Azure/azure-storage-azcopy/ste" - "github.com/Azure/azure-storage-blob-go/azblob" -) - -type syncDownloadEnumerator common.SyncJobPartOrderRequest - -// accept a new transfer, if the threshold is reached, dispatch a job part order -func (e *syncDownloadEnumerator) addTransferToUpload(transfer common.CopyTransfer, cca *cookedSyncCmdArgs) error { - - if len(e.CopyJobRequest.Transfers) == NumOfFilesPerDispatchJobPart { - resp := common.CopyJobPartOrderResponse{} - e.CopyJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // if the current part order sent to engine is 0, then set atomicSyncStatus - // variable to 1 - if e.PartNumber == 0 { - //cca.waitUntilJobCompletion(false) - cca.setFirstPartOrdered() - } - e.CopyJobRequest.Transfers = []common.CopyTransfer{} - e.PartNumber++ - } - e.CopyJobRequest.Transfers = append(e.CopyJobRequest.Transfers, transfer) - return nil -} - -// addTransferToDelete adds the filePath to the list of files to delete locally. -func (e *syncDownloadEnumerator) addTransferToDelete(filePath string) { - e.FilesToDeleteLocally = append(e.FilesToDeleteLocally, filePath) -} - -// we need to send a last part with isFinalPart set to true, along with whatever transfers that still haven't been sent -func (e *syncDownloadEnumerator) dispatchFinalPart(cca *cookedSyncCmdArgs) error { - numberOfCopyTransfers := len(e.CopyJobRequest.Transfers) - numberOfDeleteTransfers := len(e.FilesToDeleteLocally) - // If the numberoftransfer to copy / delete both are 0 - // means no transfer has been to queue to send to STE - if numberOfCopyTransfers == 0 && numberOfDeleteTransfers == 0 { - glcm.Exit("cannot start job because there are no transfer to upload or delete. "+ - "The source and destination are in sync", 0) - return nil - } - if numberOfCopyTransfers > 0 { - // Only CopyJobPart Order needs to be sent - e.CopyJobRequest.IsFinalPart = true - e.CopyJobRequest.PartNum = e.PartNumber - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // If the JobPart sent was the first part, then set atomicSyncStatus to 1, so that progress reporting can start. - if e.PartNumber == 0 { - cca.setFirstPartOrdered() - } - } - if numberOfDeleteTransfers > 0 { - answer := "" - if cca.force { - answer = "y" - } else { - answer = glcm.Prompt(fmt.Sprintf("Sync has enumerated %v files to delete locally. Do you want to delete these files ? Please confirm with y/n: ", numberOfDeleteTransfers)) - } - // read a line from stdin, if the answer is not yes, then is No, then ignore the transfers queued for deletion and continue - if !strings.EqualFold(answer, "y") { - if numberOfCopyTransfers == 0 { - glcm.Exit("cannot start job because there are no transfer to upload or delete. "+ - "The source and destination are in sync", 0) - } - cca.isEnumerationComplete = true - return nil - } - for _, file := range e.FilesToDeleteLocally { - err := os.Remove(file) - if err != nil { - glcm.Info(fmt.Sprintf("error %s deleting the file %s", err.Error(), file)) - } - } - if numberOfCopyTransfers == 0 { - glcm.Exit(fmt.Sprintf("sync completed. Deleted %v files locally ", len(e.FilesToDeleteLocally)), 0) - } - } - cca.isEnumerationComplete = true - return nil -} - -// listDestinationAndCompare lists the blob under the destination mentioned and verifies whether the blob -// exists locally or not by checking the expected localPath of blob in the sourceFiles map. If the blob -// does exists, it compares the last modified time. If it does not exists, it queues the blob for deletion. -func (e *syncDownloadEnumerator) listSourceAndCompare(cca *cookedSyncCmdArgs, p pipeline.Pipeline) error { - util := copyHandlerUtil{} - - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - // rootPath is the path of destination without wildCards - // For Example: cca.source = C:\a1\a* , so rootPath = C:\a1 - rootPath, _ := util.sourceRootPathWithoutWildCards(cca.destination) - //replace the os path separator with path separator "/" which is path separator for blobs - //sourcePattern = strings.Replace(sourcePattern, string(os.PathSeparator), "/", -1) - sourceURL, err := url.Parse(cca.source) - if err != nil { - return fmt.Errorf("error parsing the destinatio url") - } - - // since source is a remote url, it will have sas parameter - // since sas parameter will be stripped from the source url - // while cooking the raw command arguments - // source sas is added to url for listing the blobs. - sourceURL = util.appendQueryParamToUrl(sourceURL, cca.sourceSAS) - - blobUrlParts := azblob.NewBlobURLParts(*sourceURL) - blobURLPartsExtension := blobURLPartsExtension{blobUrlParts} - - containerUrl := util.getContainerUrl(blobUrlParts) - searchPrefix, pattern, _ := blobURLPartsExtension.searchPrefixFromBlobURL() - - containerBlobUrl := azblob.NewContainerURL(containerUrl, p) - - // virtual directory is the entire virtual directory path before the blob name - // passed in the searchPrefix - // Example: cca.destination = https:///vd-1? searchPrefix = vd-1/ - // virtualDirectory = vd-1 - // Example: cca.destination = https:///vd-1/vd-2/fi*.txt? searchPrefix = vd-1/vd-2/fi*.txt - // virtualDirectory = vd-1/vd-2/ - virtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) - // strip away the leading / in the closest virtual directory - if len(virtualDirectory) > 0 && virtualDirectory[0:1] == "/" { - virtualDirectory = virtualDirectory[1:] - } - - // Get the destination path without the wildcards - // This is defined since the files mentioned with exclude flag - // & include flag are relative to the Destination - // If the Destination has wildcards, then files are relative to the - // parent Destination path which is the path of last directory in the Destination - // without wildcards - // For Example: dst = "/home/user/dir1" parentSourcePath = "/home/user/dir1" - // For Example: dst = "/home/user/dir*" parentSourcePath = "/home/user" - // For Example: dst = "/home/*" parentSourcePath = "/home" - parentSourcePath := blobUrlParts.BlobName - wcIndex := util.firstIndexOfWildCard(parentSourcePath) - if wcIndex != -1 { - parentSourcePath = parentSourcePath[:wcIndex] - pathSepIndex := strings.LastIndex(parentSourcePath, common.AZCOPY_PATH_SEPARATOR_STRING) - parentSourcePath = parentSourcePath[:pathSepIndex] - } - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix - listBlob, err := containerBlobUrl.ListBlobsFlatSegment(ctx, marker, - azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) - } - - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Segment.BlobItems { - // check if the listed blob segment does not matches the sourcePath pattern - // if it does not comparison is not required - if !util.matchBlobNameAgainstPattern(pattern, blobInfo.Name, cca.recursive) { - continue - } - // realtivePathofBlobLocally is the local path relative to source at which blob should be downloaded - // Example: cca.source ="C:\User1\user-1" cca.destination = "https:///virtual-dir?" blob name = "virtual-dir/a.txt" - // realtivePathofBlobLocally = virtual-dir/a.txt - relativePathofBlobLocally := util.relativePathToRoot(parentSourcePath, blobInfo.Name, '/') - relativePathofBlobLocally = strings.Replace(relativePathofBlobLocally, virtualDirectory, "", 1) - - blobLocalPath := util.generateLocalPath(cca.destination, relativePathofBlobLocally) - - // Increment the number of files scanned at the destination. - atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) - - // calculate the expected local path of the blob - blobLocalPath = util.generateLocalPath(rootPath, relativePathofBlobLocally) - - // If the files is found in the list of files to be excluded, then it is not compared - _, found := e.SourceFilesToExclude[blobLocalPath] - if found { - continue - } - // Check if the blob exists in the map of source Files. If the file is - // found, compare the modified time of the file against the blob's last - // modified time. If the modified time of file is later than the blob's - // modified time, then queue transfer for upload. If not, then delete - // blobLocalPath from the map of sourceFiles. - localFileTime, found := e.SourceFiles[blobLocalPath] - if found { - if !blobInfo.Properties.LastModified.After(localFileTime) { - delete(e.SourceFiles, blobLocalPath) - continue - } - } - e.addTransferToUpload(common.CopyTransfer{ - Source: util.stripSASFromBlobUrl(util.generateBlobUrl(containerUrl, blobInfo.Name)).String(), - Destination: blobLocalPath, - SourceSize: *blobInfo.Properties.ContentLength, - LastModifiedTime: blobInfo.Properties.LastModified, - ContentMD5: blobInfo.Properties.ContentMD5, - }, cca) - - delete(e.SourceFiles, blobLocalPath) - } - marker = listBlob.NextMarker - } - return nil -} - -func (e *syncDownloadEnumerator) listTheDestinationIfRequired(cca *cookedSyncCmdArgs, p pipeline.Pipeline) (bool, error) { - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - util := copyHandlerUtil{} - - // attempt to parse the destination url - sourceURL, err := url.Parse(cca.source) - // the destination should have already been validated, it would be surprising if it cannot be parsed at this point - common.PanicIfErr(err) - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - sourceURL = util.appendQueryParamToUrl(sourceURL, cca.sourceSAS) - - blobUrl := azblob.NewBlobURL(*sourceURL, p) - - // Get the files and directories for the given source pattern - listOfFilesAndDir, lofaderr := filepath.Glob(cca.destination) - if lofaderr != nil { - return false, fmt.Errorf("error getting the files and directories for source pattern %s", cca.source) - } - - // Get the blob Properties - bProperties, bPropertiesError := blobUrl.GetProperties(ctx, azblob.BlobAccessConditions{}) - - // isSourceASingleFile is used to determine whether given source pattern represents single file or not - // If the source is a single file, this pointer will not be nil - // if it is nil, it means the source is a directory or list of file - var isSourceASingleFile os.FileInfo = nil - - if len(listOfFilesAndDir) == 0 { - fInfo, fError := os.Stat(listOfFilesAndDir[0]) - if fError != nil { - return false, fmt.Errorf("cannot get the information of the %s. Failed with error %s", listOfFilesAndDir[0], fError) - } - if fInfo.Mode().IsRegular() { - isSourceASingleFile = fInfo - } - } - - // sync only happens between the source and destination of same type i.e between blob and blob or between Directory and Virtual Folder / Container - // If the source is a file and destination is not a blob, sync fails. - if isSourceASingleFile != nil && bPropertiesError != nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between file %s and non blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - // If the source is a directory and destination is a blob - if isSourceASingleFile == nil && bPropertiesError == nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between directory %s and blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - - // If both source is a file and destination is a blob, then we need to do the comparison and queue the transfer if required. - if isSourceASingleFile != nil && bPropertiesError == nil { - blobName := sourceURL.Path[strings.LastIndex(sourceURL.Path, "/")+1:] - // Compare the blob name and file name - // blobName and filename should be same for sync to happen - if strings.Compare(blobName, isSourceASingleFile.Name()) != 0 { - glcm.Exit(fmt.Sprintf("sync cannot be done since blob %s and filename %s doesn't match", blobName, isSourceASingleFile.Name()), 1) - } - - // If the modified time of file local is not later than that of blob - // sync does not needs to happen. - if isSourceASingleFile.ModTime().After(bProperties.LastModified()) { - glcm.Exit(fmt.Sprintf("blob %s and file %s already in sync", blobName, isSourceASingleFile.Name()), 1) - } - - e.addTransferToUpload(common.CopyTransfer{ - Source: util.stripSASFromBlobUrl(*sourceURL).String(), - Destination: cca.source, - SourceSize: bProperties.ContentLength(), - LastModifiedTime: bProperties.LastModified(), - ContentMD5: bProperties.ContentMD5(), - }, cca) - } - - sourcePattern := "" - // Parse the source URL into blob URL parts. - blobUrlParts := azblob.NewBlobURLParts(*sourceURL) - // get the root path without wildCards and get the source Pattern - // For Example: source = /a*/*/* - // rootPath = sourcePattern = a*/*/* - blobUrlParts.BlobName, sourcePattern = util.sourceRootPathWithoutWildCards(blobUrlParts.BlobName) - - // Iterate through each file / dir inside the source - // and then checkAndQueue - for _, fileOrDir := range listOfFilesAndDir { - f, err := os.Stat(fileOrDir) - if err != nil { - glcm.Info(fmt.Sprintf("cannot get the file info for %s. failed with error %s", fileOrDir, err.Error())) - } - // directories are uploaded only if recursive is on - if f.IsDir() && cca.recursive { - // walk goes through the entire directory tree - err = filepath.Walk(fileOrDir, func(pathToFile string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if fileInfo.IsDir() { - return nil - } else { - // replace the OS path separator in pathToFile string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - pathToFile = strings.Replace(pathToFile, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - // localfileRelativePath is the path of file relative to root directory - // Example1: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\a.txt localfileRelativePath = \a.txt - // Example2: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - localfileRelativePath := strings.Replace(pathToFile, cca.destination, "", 1) - // remove the path separator at the start of relative path - if len(localfileRelativePath) > 0 && localfileRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { - localfileRelativePath = localfileRelativePath[1:] - } - // if the localfileRelativePath does not match the source pattern, then it is not compared - if !util.matchBlobNameAgainstPattern(sourcePattern, localfileRelativePath, cca.recursive) { - return nil - } - - if util.resourceShouldBeExcluded(cca.destination, e.Exclude, pathToFile) { - e.SourceFilesToExclude[pathToFile] = fileInfo.ModTime() - return nil - } - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[pathToFile] = fileInfo.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) - } - return nil - }) - } else if !f.IsDir() { - // replace the OS path separator in fileOrDir string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - fileOrDir = strings.Replace(fileOrDir, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - // localfileRelativePath is the path of file relative to root directory - // Example1: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\a.txt localfileRelativePath = \a.txt - // Example2: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - localfileRelativePath := strings.Replace(fileOrDir, cca.destination, "", 1) - // remove the path separator at the start of relative path - if len(localfileRelativePath) > 0 && localfileRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { - localfileRelativePath = localfileRelativePath[1:] - } - // if the localfileRelativePath does not match the source pattern, then it is not compared - if !util.matchBlobNameAgainstPattern(sourcePattern, localfileRelativePath, cca.recursive) { - continue - } - - if util.resourceShouldBeExcluded(cca.destination, e.Exclude, fileOrDir) { - e.SourceFilesToExclude[fileOrDir] = f.ModTime() - continue - } - - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[fileOrDir] = f.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) - } - } - return false, nil -} - -// queueSourceFilesForUpload -func (e *syncDownloadEnumerator) queueSourceFilesForUpload(cca *cookedSyncCmdArgs) { - for file, _ := range e.SourceFiles { - e.addTransferToDelete(file) - } -} - -// this function accepts the list of files/directories to transfer and processes them -func (e *syncDownloadEnumerator) enumerate(cca *cookedSyncCmdArgs) error { - ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - p, err := createBlobPipeline(ctx, e.CredentialInfo) - if err != nil { - return err - } - - // Copying the JobId of sync job to individual copyJobRequest - e.CopyJobRequest.JobID = e.JobID - // Copying the FromTo of sync job to individual copyJobRequest - e.CopyJobRequest.FromTo = e.FromTo - - // set the sas of user given Source - e.CopyJobRequest.SourceSAS = e.SourceSAS - - // set the sas of user given destination - e.CopyJobRequest.DestinationSAS = e.DestinationSAS - - // Set the preserve-last-modified-time to true in CopyJobRequest - e.CopyJobRequest.BlobAttributes.PreserveLastModifiedTime = true - - // set MD5 validation behaviour - e.CopyJobRequest.BlobAttributes.MD5ValidationOption = cca.md5ValidationOption - - // Copying the JobId of sync job to individual deleteJobRequest. - e.DeleteJobRequest.JobID = e.JobID - // FromTo of DeleteJobRequest will be BlobTrash. - e.DeleteJobRequest.FromTo = common.EFromTo.BlobTrash() - - // set the sas of user given Source - e.DeleteJobRequest.SourceSAS = e.SourceSAS - - // set the sas of user given destination - e.DeleteJobRequest.DestinationSAS = e.DestinationSAS - - // set force wriet flag to true - e.CopyJobRequest.ForceWrite = true - - //Set the log level - e.CopyJobRequest.LogLevel = e.LogLevel - e.DeleteJobRequest.LogLevel = e.LogLevel - - // Copy the sync Command String to the CopyJobPartRequest and DeleteJobRequest - e.CopyJobRequest.CommandString = e.CommandString - e.DeleteJobRequest.CommandString = e.CommandString - - // Set credential info properly - e.CopyJobRequest.CredentialInfo = e.CredentialInfo - e.DeleteJobRequest.CredentialInfo = e.CredentialInfo - - e.SourceFiles = make(map[string]time.Time) - - e.SourceFilesToExclude = make(map[string]time.Time) - - cca.waitUntilJobCompletion(false) - - isSourceABlob, err := e.listTheDestinationIfRequired(cca, p) - if err != nil { - return err - } - - // If the source provided is a blob, then remote doesn't needs to be compared against the local - // since single blob already has been compared against the file - if !isSourceABlob { - err = e.listSourceAndCompare(cca, p) - if err != nil { - return err - } - } - - e.queueSourceFilesForUpload(cca) - - // No Job Part has been dispatched, then dispatch the JobPart. - if e.PartNumber == 0 || - len(e.CopyJobRequest.Transfers) > 0 || - len(e.DeleteJobRequest.Transfers) > 0 { - err = e.dispatchFinalPart(cca) - if err != nil { - return err - } - cca.setFirstPartOrdered() - } - // scanning all the destination and is complete - cca.setScanningComplete() - return nil -} diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go new file mode 100644 index 000000000..0a668927f --- /dev/null +++ b/cmd/syncEnumerator.go @@ -0,0 +1,242 @@ +package cmd + +import ( + "errors" + "github.com/Azure/azure-storage-azcopy/common" + "strings" + "time" +) + +// -------------------------------------- Component Definitions -------------------------------------- \\ +// the following interfaces and structs allow the sync enumerator +// to be generic and has as little duplicated code as possible + +// represent a local or remote resource entity (ex: local file, blob, etc.) +// we can add more properties if needed, as this is easily extensible +type genericEntity struct { + name string + lastModifiedTime time.Time + size int64 + + // partial path relative to its directory + // example: dir=/var/a/b/c fullPath=/var/a/b/c/d/e/f.pdf relativePath=d/e/f.pdf + relativePath string +} + +func (entity *genericEntity) isMoreRecentThan(entity2 genericEntity) bool { + return entity.lastModifiedTime.After(entity2.lastModifiedTime) +} + +// capable of traversing a resource, pass each entity to the given entityProcessor if it passes all the filters +type resourceTraverser interface { + traverse(processor entityProcessor, filters []entityFilter) error +} + +// given a genericEntity, process it accordingly +type entityProcessor interface { + process(entity genericEntity) error +} + +// given a genericEntity, verify if it satisfies the defined conditions +type entityFilter interface { + pass(entity genericEntity) bool +} + +// -------------------------------------- Generic Enumerator -------------------------------------- \\ + +type syncEnumerator struct { + // these allow us to go through the source and destination + sourceTraverser resourceTraverser + destinationTraverser resourceTraverser + + // filters apply to both the source and destination + filters []entityFilter + + // the processor responsible for scheduling copy transfers + copyTransferScheduler entityProcessor + + // the processor responsible for scheduling delete transfers + deleteTransferScheduler entityProcessor + + // a finalizer that is always called if the enumeration finishes properly + finalize func() error +} + +func (e *syncEnumerator) enumerate() (err error) { + destinationIndexer := newDestinationIndexer() + + // enumerate the destination and build lookup map + err = e.destinationTraverser.traverse(destinationIndexer, e.filters) + if err != nil { + return + } + + // add the destinationIndexer as an extra filter to the list + e.filters = append(e.filters, destinationIndexer) + + // enumerate the source and schedule transfers + err = e.sourceTraverser.traverse(e.copyTransferScheduler, e.filters) + if err != nil { + return + } + + // delete extra files at the destination if needed + err = destinationIndexer.traverse(e.deleteTransferScheduler, nil) + if err != nil { + return + } + + // execute the finalize func which may perform useful clean up steps + err = e.finalize() + if err != nil { + return + } + + return +} + +// the destinationIndexer implements both entityProcessor, entityFilter, and resourceTraverser +// it is essential for the generic enumerator to work +// it can: +// 1. accumulate a lookup map with given destination entities +// 2. serve as a filter to check whether a given entity is the lookup map +// 3. go through the entities in the map like a traverser +type destinationIndexer struct { + indexMap map[string]genericEntity +} + +func newDestinationIndexer() *destinationIndexer { + indexer := destinationIndexer{} + indexer.indexMap = make(map[string]genericEntity) + + return &indexer +} + +func (i *destinationIndexer) process(entity genericEntity) (err error) { + i.indexMap[entity.relativePath] = entity + return +} + +// it will only pass items that are: +// 1. not present in the map +// 2. present but is more recent than the entry in the map +// note: we remove the entity if it is present +func (i *destinationIndexer) pass(entity genericEntity) bool { + entityInMap, present := i.indexMap[entity.relativePath] + + // if the given entity is more recent, we let it pass + if present { + defer delete(i.indexMap, entity.relativePath) + + if entity.isMoreRecentThan(entityInMap) { + return true + } + } else if !present { + return true + } + + return false +} + +// go through the entities in the map to process them +func (i *destinationIndexer) traverse(processor entityProcessor, filters []entityFilter) (err error) { + for _, value := range i.indexMap { + if !passedFilters(filters, value) { + continue + } + + err = processor.process(value) + if err != nil { + return + } + } + return +} + +// -------------------------------------- Helper Funcs -------------------------------------- \\ + +func passedFilters(filters []entityFilter, entity genericEntity) bool { + if filters != nil && len(filters) > 0 { + // loop through the filters, if any of them fail, then return false + for _, filter := range filters { + if !filter.pass(entity) { + return false + } + } + } + + return true +} + +func processIfPassedFilters(filters []entityFilter, entity genericEntity, processor entityProcessor) (err error) { + if passedFilters(filters, entity) { + err = processor.process(entity) + } + + return +} + +// entity names are useful for filters +func getEntityNameOnly(fullPath string) (nameOnly string) { + lastPathSeparator := strings.LastIndex(fullPath, common.AZCOPY_PATH_SEPARATOR_STRING) + + // if there is a path separator and it is not the last character + if lastPathSeparator > 0 && lastPathSeparator != len(fullPath)-1 { + // then we separate out the name of the entity + nameOnly = fullPath[lastPathSeparator+1:] + } else { + nameOnly = fullPath + } + + return +} + +// -------------------------------------- Implemented Enumerators -------------------------------------- \\ + +func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { + sourceTraverser, err := newBlobTraverser(cca, true) + if err != nil { + // this is unexpected + // if there is an error here, the URL was probably not valid + return nil, err + } + + destinationTraverser := newLocalTraverser(cca, false) + deleteScheduler := newSyncLocalDeleteProcessor(cca, false) + transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart) + + _, isSingleBlob := sourceTraverser.getPropertiesIfSingleBlob() + _, isSingleFile, _ := destinationTraverser.getInfoIfSingleFile() + if isSingleBlob != isSingleFile { + return nil, errors.New("sync must happen between source and destination of the same type: either blob <-> file, or container/virtual directory <-> local directory") + } + + finalize := func() error { + jobInitiated, err := transferScheduler.dispatchFinalPart() + if err != nil { + return err + } + + if !jobInitiated && !deleteScheduler.wasAnyFileDeleted() { + return errors.New("the source and destination are already in sync") + } + + cca.setScanningComplete() + return nil + } + + includeFilters := buildIncludeFilters(cca.include) + excludeFilters := buildExcludeFilters(cca.exclude) + + // trigger the progress reporting + cca.waitUntilJobCompletion(false) + + return &syncEnumerator{ + sourceTraverser: sourceTraverser, + destinationTraverser: destinationTraverser, + copyTransferScheduler: transferScheduler, + deleteTransferScheduler: deleteScheduler, + finalize: finalize, + filters: append(includeFilters, excludeFilters...), + }, nil +} diff --git a/cmd/syncFilter.go b/cmd/syncFilter.go new file mode 100644 index 000000000..aa0c5d741 --- /dev/null +++ b/cmd/syncFilter.go @@ -0,0 +1,71 @@ +package cmd + +import "path" + +type excludeFilter struct { + pattern string +} + +func (f *excludeFilter) pass(entity genericEntity) bool { + matched, err := path.Match(f.pattern, entity.name) + + // if the pattern failed to match with an error, then we assume the pattern is invalid + // and let it pass + if err != nil { + return true + } + + if matched { + return false + } + + return true +} + +func buildExcludeFilters(patterns []string) []entityFilter { + filters := make([]entityFilter, 0) + for _, pattern := range patterns { + filters = append(filters, &excludeFilter{pattern: pattern}) + } + + return filters +} + +// design explanation: +// include filters are different from the exclude ones, which work together in the "AND" manner +// meaning and if an entity is rejected by any of the exclude filters, then it is rejected by all of them +// as a result, the exclude filters can be in their own struct, and work correctly +// on the other hand, include filters work in the "OR" manner +// meaning that if an entity is accepted by any of the include filters, then it is accepted by all of them +// consequently, all the include patterns must be stored together +type includeFilter struct { + patterns []string +} + +func (f *includeFilter) pass(entity genericEntity) bool { + if len(f.patterns) == 0 { + return true + } + + for _, pattern := range f.patterns { + matched, err := path.Match(pattern, entity.name) + + // if the pattern failed to match with an error, then we assume the pattern is invalid + // and ignore it + if err != nil { + continue + } + + // if an entity is accepted by any of the include filters + // it is accepted + if matched { + return true + } + } + + return false +} + +func buildIncludeFilters(patterns []string) []entityFilter { + return []entityFilter{&includeFilter{patterns: patterns}} +} diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go new file mode 100644 index 000000000..de6a7efe3 --- /dev/null +++ b/cmd/syncProcessor.go @@ -0,0 +1,181 @@ +package cmd + +import ( + "fmt" + "github.com/Azure/azure-storage-azcopy/common" + "os" + "path/filepath" + "strings" +) + +type syncTransferProcessor struct { + numOfTransfersPerPart int + copyJobTemplate *common.CopyJobPartOrderRequest + source string + destination string + + // keep a handle to initiate progress tracking + cca *cookedSyncCmdArgs +} + +func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) *syncTransferProcessor { + processor := syncTransferProcessor{} + processor.copyJobTemplate = &common.CopyJobPartOrderRequest{ + JobID: cca.jobID, + CommandString: cca.commandString, + FromTo: cca.fromTo, + + // authentication related + CredentialInfo: cca.credentialInfo, + SourceSAS: cca.sourceSAS, + DestinationSAS: cca.destinationSAS, + + // flags + BlobAttributes: common.BlobTransferAttributes{ + PreserveLastModifiedTime: true, + MD5ValidationOption: cca.md5ValidationOption, + }, + ForceWrite: true, + LogLevel: cca.logVerbosity, + } + + // useful for building transfers later + processor.source = cca.source + processor.destination = cca.destination + + processor.cca = cca + processor.numOfTransfersPerPart = numOfTransfersPerPart + return &processor +} + +func (s *syncTransferProcessor) process(entity genericEntity) (err error) { + if len(s.copyJobTemplate.Transfers) == s.numOfTransfersPerPart { + err = s.sendPartToSte() + if err != nil { + return err + } + + // reset the transfers buffer + s.copyJobTemplate.Transfers = []common.CopyTransfer{} + s.copyJobTemplate.PartNum++ + } + + // only append the transfer after we've checked and dispatched a part + // so that there is at least one transfer for the final part + s.copyJobTemplate.Transfers = append(s.copyJobTemplate.Transfers, common.CopyTransfer{ + Source: s.appendEntityPathToResourcePath(entity.relativePath, s.source), + Destination: s.appendEntityPathToResourcePath(entity.relativePath, s.destination), + SourceSize: entity.size, + LastModifiedTime: entity.lastModifiedTime, + }) + return nil +} + +func (s *syncTransferProcessor) appendEntityPathToResourcePath(entityPath, parentPath string) string { + if entityPath == "" { + return parentPath + } + + return strings.Join([]string{parentPath, entityPath}, common.AZCOPY_PATH_SEPARATOR_STRING) +} + +func (s *syncTransferProcessor) dispatchFinalPart() (copyJobInitiated bool, err error) { + numberOfCopyTransfers := len(s.copyJobTemplate.Transfers) + + // if the number of transfer to copy is + // and no part was dispatched, then it means there is no work to do + if s.copyJobTemplate.PartNum == 0 && numberOfCopyTransfers == 0 { + return false, nil + } + + if numberOfCopyTransfers > 0 { + s.copyJobTemplate.IsFinalPart = true + err = s.sendPartToSte() + if err != nil { + return false, err + } + } + + s.cca.isEnumerationComplete = true + return true, nil +} + +func (s *syncTransferProcessor) sendPartToSte() error { + var resp common.CopyJobPartOrderResponse + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(s.copyJobTemplate), &resp) + if !resp.JobStarted { + return fmt.Errorf("copy job part order with JobId %s and part number %d failed to dispatch because %s", + s.copyJobTemplate.JobID, s.copyJobTemplate.PartNum, resp.ErrorMsg) + } + + // if the current part order sent to ste is 0, then alert the progress reporting routine + if s.copyJobTemplate.PartNum == 0 { + s.cca.setFirstPartOrdered() + } + + return nil +} + +type syncLocalDeleteProcessor struct { + rootPath string + + // ask the user for permission the first time we delete a file + hasPromptedUser bool + + // note down whether any delete should happen + shouldDelete bool + + // keep a handle for progress tracking + cca *cookedSyncCmdArgs +} + +func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs, isSource bool) *syncLocalDeleteProcessor { + rootPath := cca.source + if !isSource { + rootPath = cca.destination + } + + return &syncLocalDeleteProcessor{rootPath: rootPath, cca: cca, hasPromptedUser: false} +} + +func (s *syncLocalDeleteProcessor) process(entity genericEntity) (err error) { + if !s.hasPromptedUser { + s.shouldDelete = s.promptForConfirmation() + } + + if !s.shouldDelete { + return nil + } + + err = os.Remove(filepath.Join(s.rootPath, entity.relativePath)) + if err != nil { + glcm.Info(fmt.Sprintf("error %s deleting the file %s", err.Error(), entity.relativePath)) + } + + return +} + +func (s *syncLocalDeleteProcessor) promptForConfirmation() (shouldDelete bool) { + shouldDelete = false + + // omit asking if the user has already specified + if s.cca.force { + shouldDelete = true + } else { + answer := glcm.Prompt(fmt.Sprintf("Sync has discovered local files that are not present at the source, would you like to delete them? Please confirm with y/n: ")) + if answer == "y" || answer == "yes" { + shouldDelete = true + glcm.Info("Confirmed. The extra local files will be deleted.") + } else { + glcm.Info("No deletions will happen.") + } + } + + s.hasPromptedUser = true + return +} + +func (s *syncLocalDeleteProcessor) wasAnyFileDeleted() bool { + // we'd have prompted the user if any entity was passed in + return s.hasPromptedUser +} diff --git a/cmd/syncTraverser.go b/cmd/syncTraverser.go new file mode 100644 index 000000000..6dd2ef6ea --- /dev/null +++ b/cmd/syncTraverser.go @@ -0,0 +1,268 @@ +package cmd + +import ( + "context" + "fmt" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-azcopy/ste" + "github.com/Azure/azure-storage-blob-go/azblob" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" +) + +// -------------------------------------- Traversers -------------------------------------- \\ +// these traversers allow us to iterate through different resource types + +type blobTraverser struct { + rawURL *url.URL + p pipeline.Pipeline + ctx context.Context + recursive bool + isSource bool + cca *cookedSyncCmdArgs +} + +func (blobTraverser *blobTraverser) getPropertiesIfSingleBlob() (blobProps *azblob.BlobGetPropertiesResponse, isBlob bool) { + blobURL := azblob.NewBlobURL(*blobTraverser.rawURL, blobTraverser.p) + blobProps, blobPropertiesErr := blobURL.GetProperties(blobTraverser.ctx, azblob.BlobAccessConditions{}) + + // if there was no problem getting the properties, it means that we are looking at a single blob + if blobPropertiesErr == nil { + isBlob = true + return + } + + return +} + +func (blobTraverser *blobTraverser) traverse(processor entityProcessor, filters []entityFilter) (err error) { + blobUrlParts := azblob.NewBlobURLParts(*blobTraverser.rawURL) + + // check if the url points to a single blob + blobProperties, isBlob := blobTraverser.getPropertiesIfSingleBlob() + if isBlob { + entity := genericEntity{ + name: getEntityNameOnly(blobUrlParts.BlobName), + relativePath: "", // relative path makes no sense when the full path already points to the file + lastModifiedTime: blobProperties.LastModified(), + size: blobProperties.ContentLength(), + } + blobTraverser.incrementEnumerationCounter() + return processIfPassedFilters(filters, entity, processor) + } + + // get the container URL so that we can list the blobs + containerRawURL := copyHandlerUtil{}.getContainerUrl(blobUrlParts) + containerURL := azblob.NewContainerURL(containerRawURL, blobTraverser.p) + + // get the search prefix to aid in the listing + searchPrefix := blobUrlParts.BlobName + + // append a slash if it is not already present + if searchPrefix != "" && !strings.HasSuffix(searchPrefix, common.AZCOPY_PATH_SEPARATOR_STRING) { + searchPrefix += common.AZCOPY_PATH_SEPARATOR_STRING + } + + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix + listBlob, err := containerURL.ListBlobsFlatSegment(blobTraverser.ctx, marker, + azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) + if err != nil { + return fmt.Errorf("cannot list blobs. Failed with error %s", err.Error()) + } + + // process the blobs returned in this result segment + for _, blobInfo := range listBlob.Segment.BlobItems { + relativePath := strings.Replace(blobInfo.Name, searchPrefix, "", 1) + + // if recursive + if !blobTraverser.recursive && strings.Contains(relativePath, common.AZCOPY_PATH_SEPARATOR_STRING) { + continue + } + + entity := genericEntity{ + name: getEntityNameOnly(blobInfo.Name), + relativePath: relativePath, + lastModifiedTime: blobInfo.Properties.LastModified, + size: *blobInfo.Properties.ContentLength, + } + blobTraverser.incrementEnumerationCounter() + processErr := processIfPassedFilters(filters, entity, processor) + if processErr != nil { + return processErr + } + } + + marker = listBlob.NextMarker + } + + return +} + +func (blobTraverser *blobTraverser) incrementEnumerationCounter() { + var counterAddr *uint64 + + if blobTraverser.isSource { + counterAddr = &blobTraverser.cca.atomicSourceFilesScanned + } else { + counterAddr = &blobTraverser.cca.atomicDestinationFilesScanned + } + + atomic.AddUint64(counterAddr, 1) +} + +func newBlobTraverser(cca *cookedSyncCmdArgs, isSource bool) (traverser *blobTraverser, err error) { + traverser = &blobTraverser{} + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + + if isSource { + traverser.rawURL, err = url.Parse(cca.source) + if err == nil && cca.sourceSAS != "" { + copyHandlerUtil{}.appendQueryParamToUrl(traverser.rawURL, cca.sourceSAS) + } + + } else { + traverser.rawURL, err = url.Parse(cca.destination) + if err == nil && cca.destinationSAS != "" { + copyHandlerUtil{}.appendQueryParamToUrl(traverser.rawURL, cca.destinationSAS) + } + } + + if err != nil { + return + } + + traverser.p, err = createBlobPipeline(ctx, cca.credentialInfo) + if err != nil { + return + } + + traverser.isSource = isSource + traverser.ctx = context.TODO() + traverser.recursive = cca.recursive + traverser.cca = cca + return +} + +type localTraverser struct { + fullPath string + recursive bool + followSymlinks bool + isSource bool + cca *cookedSyncCmdArgs +} + +func (localTraverser *localTraverser) traverse(processor entityProcessor, filters []entityFilter) (err error) { + singleFileInfo, isSingleFile, err := localTraverser.getInfoIfSingleFile() + + if err != nil { + return fmt.Errorf("cannot scan the path %s, please verify that it is a valid", localTraverser.fullPath) + } + + // if the path is a single file, then pass it through the filters and send to processor + if isSingleFile { + localTraverser.incrementEnumerationCounter() + err = processIfPassedFilters(filters, genericEntity{ + name: singleFileInfo.Name(), + relativePath: "", // relative path makes no sense when the full path already points to the file + lastModifiedTime: singleFileInfo.ModTime(), + size: singleFileInfo.Size()}, processor) + return + + } else { + if localTraverser.recursive { + err = filepath.Walk(localTraverser.fullPath, func(filePath string, fileInfo os.FileInfo, fileError error) error { + if fileError != nil { + return fileError + } + + // skip the subdirectories + if fileInfo.IsDir() { + return nil + } + + localTraverser.incrementEnumerationCounter() + return processIfPassedFilters(filters, genericEntity{ + name: fileInfo.Name(), + relativePath: strings.Replace(filePath, localTraverser.fullPath+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1), + lastModifiedTime: fileInfo.ModTime(), + size: fileInfo.Size()}, processor) + }) + + return + } else { + // if recursive is off, we only need to scan the files immediately under the fullPath + files, err := ioutil.ReadDir(localTraverser.fullPath) + if err != nil { + return err + } + + // go through the files and return if any of them fail to process + for _, singleFile := range files { + if singleFile.IsDir() { + continue + } + + localTraverser.incrementEnumerationCounter() + err = processIfPassedFilters(filters, genericEntity{ + name: singleFile.Name(), + relativePath: singleFile.Name(), + lastModifiedTime: singleFile.ModTime(), + size: singleFile.Size()}, processor) + + if err != nil { + return err + } + } + } + } + + return +} + +func (localTraverser *localTraverser) getInfoIfSingleFile() (os.FileInfo, bool, error) { + fileInfo, err := os.Stat(localTraverser.fullPath) + + if err != nil { + return nil, false, err + } + + if fileInfo.IsDir() { + return nil, false, nil + } + + return fileInfo, true, nil +} + +func (localTraverser *localTraverser) incrementEnumerationCounter() { + var counterAddr *uint64 + + if localTraverser.isSource { + counterAddr = &localTraverser.cca.atomicSourceFilesScanned + } else { + counterAddr = &localTraverser.cca.atomicDestinationFilesScanned + } + + atomic.AddUint64(counterAddr, 1) +} + +func newLocalTraverser(cca *cookedSyncCmdArgs, isSource bool) *localTraverser { + traverser := localTraverser{} + + if isSource { + traverser.fullPath = cca.source + } else { + traverser.fullPath = cca.destination + } + + traverser.isSource = isSource + traverser.recursive = cca.recursive + traverser.followSymlinks = cca.followSymlinks + traverser.cca = cca + return &traverser +} diff --git a/cmd/syncUploadEnumerator.go b/cmd/syncUploadEnumerator.go deleted file mode 100644 index 249cc155e..000000000 --- a/cmd/syncUploadEnumerator.go +++ /dev/null @@ -1,563 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - "sync/atomic" - "time" - - "github.com/Azure/azure-pipeline-go/pipeline" - "github.com/Azure/azure-storage-azcopy/common" - "github.com/Azure/azure-storage-azcopy/ste" - "github.com/Azure/azure-storage-blob-go/azblob" -) - -type syncUploadEnumerator common.SyncJobPartOrderRequest - -// accepts a new transfer which is to delete the blob on container. -func (e *syncUploadEnumerator) addTransferToDelete(transfer common.CopyTransfer, cca *cookedSyncCmdArgs) error { - e.DeleteJobRequest.Transfers = append(e.DeleteJobRequest.Transfers, transfer) - return nil -} - -// accept a new transfer, if the threshold is reached, dispatch a job part order -func (e *syncUploadEnumerator) addTransferToUpload(transfer common.CopyTransfer, cca *cookedSyncCmdArgs) error { - - if len(e.CopyJobRequest.Transfers) == NumOfFilesPerDispatchJobPart { - resp := common.CopyJobPartOrderResponse{} - e.CopyJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // if the current part order sent to engine is 0, then start fetching the Job Progress summary. - if e.PartNumber == 0 { - //update this atomic counter which is monitored by another go routine - //reporting numbers to the user - cca.setFirstPartOrdered() - } - e.CopyJobRequest.Transfers = []common.CopyTransfer{} - e.PartNumber++ - } - e.CopyJobRequest.Transfers = append(e.CopyJobRequest.Transfers, transfer) - return nil -} - -// we need to send a last part with isFinalPart set to true, along with whatever transfers that still haven't been sent -func (e *syncUploadEnumerator) dispatchFinalPart(cca *cookedSyncCmdArgs) error { - numberOfCopyTransfers := uint64(len(e.CopyJobRequest.Transfers)) - numberOfDeleteTransfers := uint64(len(e.DeleteJobRequest.Transfers)) - if numberOfCopyTransfers == 0 && numberOfDeleteTransfers == 0 { - glcm.Exit("cannot start job because there are no files to upload or delete. "+ - "The source and destination are in sync", 0) - return nil - } - // sendDeleteTransfers is an internal function which creates JobPartRequest for all the delete transfers enumerated. - // It creates requests for group of 10000 transfers. - sendDeleteTransfers := func() error { - currentCount := uint64(0) - // If the user agrees to delete the transfers, then break the entire deleteJobRequest into parts of 10000 size and send them - for numberOfDeleteTransfers > 0 { - // number of transfers in the current request can be either 10,000 or less than that. - numberOfTransfers := common.Iffuint64(numberOfDeleteTransfers > NumOfFilesPerDispatchJobPart, NumOfFilesPerDispatchJobPart, numberOfDeleteTransfers) - // create a copy of DeleteJobRequest - deleteJobRequest := e.DeleteJobRequest - // Reset the transfer list in the copy of DeleteJobRequest - deleteJobRequest.Transfers = []common.CopyTransfer{} - // Copy the transfers from currentCount till number of transfer calculated for current iteration - deleteJobRequest.Transfers = e.DeleteJobRequest.Transfers[currentCount : currentCount+numberOfTransfers] - // Set the part number - deleteJobRequest.PartNum = e.PartNumber - // Increment the part number - e.PartNumber++ - // Increment the current count - currentCount += numberOfTransfers - // Decrease the numberOfDeleteTransfer by the number Of transfers calculated for the current iteration - numberOfDeleteTransfers -= numberOfTransfers - // If the number of delete transfer is 0, it means it is the last part that needs to be sent. - // Set the IsFinalPart for the current request to true - if numberOfDeleteTransfers == 0 { - deleteJobRequest.IsFinalPart = true - } - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&deleteJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // If the part sent above was the first, then set setFirstPartOrdered, so that progress can be fetched. - if deleteJobRequest.PartNum == 0 { - cca.setFirstPartOrdered() - } - } - return nil - } - if numberOfCopyTransfers > 0 && numberOfDeleteTransfers > 0 { - var resp common.CopyJobPartOrderResponse - e.CopyJobRequest.PartNum = e.PartNumber - answer := "" - if cca.force { - answer = "y" - } else { - answer = glcm.Prompt(fmt.Sprintf("Sync has enumerated %v files to delete from destination. Do you want to delete these files ? Please confirm with y/n: ", numberOfDeleteTransfers)) - } - // read a line from stdin, if the answer is not yes, then is No, then ignore the transfers queued for deletion and continue - if !strings.EqualFold(answer, "y") { - e.CopyJobRequest.IsFinalPart = true - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - return nil - } - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // If the part sent above was the first, then setFirstPartOrdered, so that progress can be fetched. - if e.PartNumber == 0 { - cca.setFirstPartOrdered() - } - e.PartNumber++ - err := sendDeleteTransfers() - cca.isEnumerationComplete = true - return err - } else if numberOfCopyTransfers > 0 { - e.CopyJobRequest.IsFinalPart = true - e.CopyJobRequest.PartNum = e.PartNumber - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - cca.isEnumerationComplete = true - return nil - } - answer := "" - // If the user set the force flag to true, then prompt is not required and file will be deleted. - if cca.force { - answer = "y" - } else { - answer = glcm.Prompt(fmt.Sprintf("Sync has enumerated %v files to delete from destination. Do you want to delete these files ? Please confirm with y/n: ", numberOfDeleteTransfers)) - } - // read a line from stdin, if the answer is not yes, then is No, then ignore the transfers queued for deletion and continue - if !strings.EqualFold(answer, "y") { - return fmt.Errorf("cannot start job because there are no transfer to upload or delete. " + - "The source and destination are in sync") - } - error := sendDeleteTransfers() - cca.isEnumerationComplete = true - return error -} - -func (e *syncUploadEnumerator) listTheSourceIfRequired(cca *cookedSyncCmdArgs, p pipeline.Pipeline) (bool, error) { - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - util := copyHandlerUtil{} - - // attempt to parse the destination url - destinationUrl, err := url.Parse(cca.destination) - // the destination should have already been validated, it would be surprising if it cannot be parsed at this point - common.PanicIfErr(err) - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - destinationUrl = util.appendQueryParamToUrl(destinationUrl, cca.destinationSAS) - - blobUrl := azblob.NewBlobURL(*destinationUrl, p) - - // Get the files and directories for the given source pattern - listOfFilesAndDir, lofaderr := filepath.Glob(cca.source) - if lofaderr != nil { - return false, fmt.Errorf("error getting the files and directories for source pattern %s", cca.source) - } - - // Get the blob Properties - bProperties, bPropertiesError := blobUrl.GetProperties(ctx, azblob.BlobAccessConditions{}) - - // isSourceASingleFile is used to determine whether given source pattern represents single file or not - // If the source is a single file, this pointer will not be nil - // if it is nil, it means the source is a directory or list of file - var isSourceASingleFile os.FileInfo = nil - - if len(listOfFilesAndDir) == 0 { - fInfo, fError := os.Stat(listOfFilesAndDir[0]) - if fError != nil { - return false, fmt.Errorf("cannot get the information of the %s. Failed with error %s", listOfFilesAndDir[0], fError) - } - if fInfo.Mode().IsRegular() { - isSourceASingleFile = fInfo - } - } - - // sync only happens between the source and destination of same type i.e between blob and blob or between Directory and Virtual Folder / Container - // If the source is a file and destination is not a blob, sync fails. - if isSourceASingleFile != nil && bPropertiesError != nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between file %s and non blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - // If the source is a directory and destination is a blob - if isSourceASingleFile == nil && bPropertiesError == nil { - glcm.Exit(fmt.Sprintf("Cannot perform sync between directory %s and blob destination %s. sync only happens between source and destination of same type", cca.source, cca.destination), 1) - } - - // If both source is a file and destination is a blob, then we need to do the comparison and queue the transfer if required. - if isSourceASingleFile != nil && bPropertiesError == nil { - blobName := destinationUrl.Path[strings.LastIndex(destinationUrl.Path, "/")+1:] - // Compare the blob name and file name - // blobName and filename should be same for sync to happen - if strings.Compare(blobName, isSourceASingleFile.Name()) != 0 { - glcm.Exit(fmt.Sprintf("sync cannot be done since blob %s and filename %s doesn't match", blobName, isSourceASingleFile.Name()), 1) - } - - // If the modified time of file local is not later than that of blob - // sync does not needs to happen. - if !isSourceASingleFile.ModTime().After(bProperties.LastModified()) { - glcm.Exit(fmt.Sprintf("blob %s and file %s already in sync", blobName, isSourceASingleFile.Name()), 1) - } - - e.addTransferToUpload(common.CopyTransfer{ - Source: cca.source, - Destination: util.stripSASFromBlobUrl(*destinationUrl).String(), - SourceSize: isSourceASingleFile.Size(), - LastModifiedTime: isSourceASingleFile.ModTime(), - }, cca) - return true, nil - } - - if len(listOfFilesAndDir) == 1 && !cca.recursive { - glcm.Exit(fmt.Sprintf("error performing between source %s and destination %s is a directory. recursive flag is turned off.", cca.source, cca.destination), 1) - } - // Get the source path without the wildcards - // This is defined since the files mentioned with exclude flag - // & include flag are relative to the Source - // If the source has wildcards, then files are relative to the - // parent source path which is the path of last directory in the source - // without wildcards - // For Example: src = "/home/user/dir1" parentSourcePath = "/home/user/dir1" - // For Example: src = "/home/user/dir*" parentSourcePath = "/home/user" - // For Example: src = "/home/*" parentSourcePath = "/home" - parentSourcePath := cca.source - wcIndex := util.firstIndexOfWildCard(parentSourcePath) - if wcIndex != -1 { - parentSourcePath = parentSourcePath[:wcIndex] - pathSepIndex := strings.LastIndex(parentSourcePath, common.AZCOPY_PATH_SEPARATOR_STRING) - parentSourcePath = parentSourcePath[:pathSepIndex] - } - - // Iterate through each file / dir inside the source - // and then checkAndQueue - for _, fileOrDir := range listOfFilesAndDir { - f, err := os.Stat(fileOrDir) - if err != nil { - glcm.Info(fmt.Sprintf("cannot get the file Info for %s. failed with error %s", fileOrDir, err.Error())) - } - // directories are uploaded only if recursive is on - if f.IsDir() && cca.recursive { - // walk goes through the entire directory tree - err = filepath.Walk(fileOrDir, func(pathToFile string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if fileInfo.IsDir() { - return nil - } else { - // replace the OS path separator in pathToFile string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - pathToFile = strings.Replace(pathToFile, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - if util.resourceShouldBeExcluded(parentSourcePath, e.Exclude, pathToFile) { - e.SourceFilesToExclude[pathToFile] = f.ModTime() - return nil - } - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[pathToFile] = fileInfo.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) - } - return nil - }) - } else if !f.IsDir() { - // replace the OS path separator in fileOrDir string with AZCOPY_PATH_SEPARATOR - // this replacement is done to handle the windows file paths where path separator "\\" - fileOrDir = strings.Replace(fileOrDir, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - - if util.resourceShouldBeExcluded(parentSourcePath, e.Exclude, fileOrDir) { - e.SourceFilesToExclude[fileOrDir] = f.ModTime() - continue - } - if len(e.SourceFiles) > MaxNumberOfFilesAllowedInSync { - glcm.Exit(fmt.Sprintf("cannot sync the source %s with more than %v number of files", cca.source, MaxNumberOfFilesAllowedInSync), 1) - } - e.SourceFiles[fileOrDir] = f.ModTime() - // Increment the sync counter. - atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) - } - } - return false, nil -} - -// listDestinationAndCompare lists the blob under the destination mentioned and verifies whether the blob -// exists locally or not by checking the expected localPath of blob in the sourceFiles map. If the blob -// does exists, it compares the last modified time. If it does not exists, it queues the blob for deletion. -func (e *syncUploadEnumerator) listDestinationAndCompare(cca *cookedSyncCmdArgs, p pipeline.Pipeline) error { - util := copyHandlerUtil{} - - ctx := context.WithValue(context.Background(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - // rootPath is the path of source without wildCards - // sourcePattern is the filePath pattern inside the source - // For Example: cca.source = C:\a1\a* , so rootPath = C:\a1 and filePattern is a* - // This is to avoid enumerator to compare any file inside the destination directory - // that doesn't match the pattern - // For Example: cca.source = C:\a1\a* des = https://? - // Only files that follow pattern a* will be compared - rootPath, sourcePattern := util.sourceRootPathWithoutWildCards(cca.source) - //replace the os path separator with path separator "/" which is path separator for blobs - //sourcePattern = strings.Replace(sourcePattern, string(os.PathSeparator), "/", -1) - destinationUrl, err := url.Parse(cca.destination) - if err != nil { - return fmt.Errorf("error parsing the destinatio url") - } - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - destinationUrl = util.appendQueryParamToUrl(destinationUrl, cca.destinationSAS) - - blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) - blobURLPartsExtension := blobURLPartsExtension{blobUrlParts} - - containerUrl := util.getContainerUrl(blobUrlParts) - searchPrefix, _, _ := blobURLPartsExtension.searchPrefixFromBlobURL() - - containerBlobUrl := azblob.NewContainerURL(containerUrl, p) - - // Get the destination path without the wildcards - // This is defined since the files mentioned with exclude flag - // & include flag are relative to the Destination - // If the Destination has wildcards, then files are relative to the - // parent Destination path which is the path of last directory in the Destination - // without wildcards - // For Example: dst = "/home/user/dir1" parentSourcePath = "/home/user/dir1" - // For Example: dst = "/home/user/dir*" parentSourcePath = "/home/user" - // For Example: dst = "/home/*" parentSourcePath = "/home" - parentDestinationPath := blobUrlParts.BlobName - wcIndex := util.firstIndexOfWildCard(parentDestinationPath) - if wcIndex != -1 { - parentDestinationPath = parentDestinationPath[:wcIndex] - pathSepIndex := strings.LastIndex(parentDestinationPath, common.AZCOPY_PATH_SEPARATOR_STRING) - parentDestinationPath = parentDestinationPath[:pathSepIndex] - } - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix - listBlob, err := containerBlobUrl.ListBlobsFlatSegment(ctx, marker, - azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) - } - - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Segment.BlobItems { - // realtivePathofBlobLocally is the local path relative to source at which blob should be downloaded - // Example: cca.source ="C:\User1\user-1" cca.destination = "https:///virtual-dir?" blob name = "virtual-dir/a.txt" - // realtivePathofBlobLocally = virtual-dir/a.txt - realtivePathofBlobLocally := util.relativePathToRoot(searchPrefix, blobInfo.Name, '/') - - // check if the listed blob segment matches the sourcePath pattern - // if it does not comparison is not required - if !util.matchBlobNameAgainstPattern(sourcePattern, realtivePathofBlobLocally, cca.recursive) { - continue - } - - // Increment the number of files scanned at the destination. - atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) - - // calculate the expected local path of the blob - blobLocalPath := util.generateLocalPath(rootPath, realtivePathofBlobLocally) - - // If the files is found in the list of files to be excluded, then it is not compared - _, found := e.SourceFilesToExclude[blobLocalPath] - if found { - continue - } - // Check if the blob exists in the map of source Files. If the file is - // found, compare the modified time of the file against the blob's last - // modified time. If the modified time of file is later than the blob's - // modified time, then queue transfer for upload. If not, then delete - // blobLocalPath from the map of sourceFiles. - localFileTime, found := e.SourceFiles[blobLocalPath] - if found { - if localFileTime.After(blobInfo.Properties.LastModified) { - e.addTransferToUpload(common.CopyTransfer{ - Source: blobLocalPath, - Destination: util.stripSASFromBlobUrl(util.generateBlobUrl(containerUrl, blobInfo.Name)).String(), - SourceSize: *blobInfo.Properties.ContentLength, - }, cca) - } - delete(e.SourceFiles, blobLocalPath) - } else { - // If the blob is not found in the map of source Files, queue it for - // delete - e.addTransferToDelete(common.CopyTransfer{ - Source: util.stripSASFromBlobUrl(util.generateBlobUrl(containerUrl, blobInfo.Name)).String(), - Destination: "", // no destination in case of Delete JobPartOrder - SourceSize: *blobInfo.Properties.ContentLength, - }, cca) - } - } - marker = listBlob.NextMarker - } - return nil -} - -// queueSourceFilesForUpload -func (e *syncUploadEnumerator) queueSourceFilesForUpload(cca *cookedSyncCmdArgs) { - util := copyHandlerUtil{} - // rootPath will be the parent source directory before the first wildcard - // For Example: cca.source = C:\a\b* rootPath = C:\a - // For Example: cca.source = C:\*\a* rootPath = c:\ - // In case of no wildCard, rootPath is equal to the source directory - // This rootPath is effective when wildCards are provided - // Using this rootPath, path of file on blob is calculated - rootPath, _ := util.sourceRootPathWithoutWildCards(cca.source) - - // attempt to parse the destination url - destinationUrl, err := url.Parse(cca.destination) - // the destination should have already been validated, it would be surprising if it cannot be parsed at this point - common.PanicIfErr(err) - - // since destination is a remote url, it will have sas parameter - // since sas parameter will be stripped from the destination url - // while cooking the raw command arguments - // destination sas is added to url for listing the blobs. - destinationUrl = util.appendQueryParamToUrl(destinationUrl, cca.destinationSAS) - - blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) - - for file, _ := range e.SourceFiles { - // get the file Info - f, err := os.Stat(file) - if err != nil { - glcm.Info(fmt.Sprintf("Error %s getting the file info for file %s", err.Error(), file)) - continue - } - // localfileRelativePath is the path of file relative to root directory - // Example1: rootPath = C:\User\user1\dir-1 fileAbsolutePath = C:\User\user1\dir-1\a.txt localfileRelativePath = \a.txt - // Example2: rootPath = C:\User\user1\dir-1 fileAbsolutePath = C:\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - localfileRelativePath := strings.Replace(file, rootPath, "", 1) - // remove the path separator at the start of relative path - if len(localfileRelativePath) > 0 && localfileRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { - localfileRelativePath = localfileRelativePath[1:] - } - // Appending the fileRelativePath to the destinationUrl - // root = C:\User\user1\dir-1 cca.destination = https:///? - // fileAbsolutePath = C:\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt - // filedestinationUrl = https:////dir-2/a.txt? - filedestinationUrl, _ := util.appendBlobNameToUrl(blobUrlParts, localfileRelativePath) - - err = e.addTransferToUpload(common.CopyTransfer{ - Source: file, - Destination: util.stripSASFromBlobUrl(filedestinationUrl).String(), - LastModifiedTime: f.ModTime(), - SourceSize: f.Size(), - }, cca) - if err != nil { - glcm.Info(fmt.Sprintf("Error %s uploading transfer source :%s and destination %s", err.Error(), file, util.stripSASFromBlobUrl(filedestinationUrl).String())) - } - - } -} - -// this function accepts the list of files/directories to transfer and processes them -func (e *syncUploadEnumerator) enumerate(cca *cookedSyncCmdArgs) error { - ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) - - // Create the new azblob pipeline - p, err := createBlobPipeline(ctx, e.CredentialInfo) - if err != nil { - return err - } - - // Copying the JobId of sync job to individual copyJobRequest - e.CopyJobRequest.JobID = e.JobID - // Copying the FromTo of sync job to individual copyJobRequest - e.CopyJobRequest.FromTo = e.FromTo - - // set the sas of user given Source - e.CopyJobRequest.SourceSAS = e.SourceSAS - - // set the sas of user given destination - e.CopyJobRequest.DestinationSAS = e.DestinationSAS - - // Copying the JobId of sync job to individual deleteJobRequest. - e.DeleteJobRequest.JobID = e.JobID - // FromTo of DeleteJobRequest will be BlobTrash. - e.DeleteJobRequest.FromTo = common.EFromTo.BlobTrash() - - // For delete the source is the destination in case of sync upload - // For Example: source = /home/user destination = https://container/vd-1? - // For deleting the blobs, Source in Delete Job Source will be the blob url - // and source sas is the destination sas which is url sas. - // set the destination sas as the source sas - e.DeleteJobRequest.SourceSAS = e.DestinationSAS - - // Set the Log Level - e.CopyJobRequest.LogLevel = e.LogLevel - e.DeleteJobRequest.LogLevel = e.LogLevel - - // Set the force flag to true - e.CopyJobRequest.ForceWrite = true - - // Copy the sync Command String to the CopyJobPartRequest and DeleteJobRequest - e.CopyJobRequest.CommandString = e.CommandString - e.DeleteJobRequest.CommandString = e.CommandString - - // Set credential info properly - e.CopyJobRequest.CredentialInfo = e.CredentialInfo - e.DeleteJobRequest.CredentialInfo = e.CredentialInfo - - e.SourceFiles = make(map[string]time.Time) - - e.SourceFilesToExclude = make(map[string]time.Time) - - cca.waitUntilJobCompletion(false) - - // list the source files and store in the map. - // While listing the source files, it applies the exclude filter - // and stores them into a separate map "sourceFilesToExclude" - isSourceAFile, err := e.listTheSourceIfRequired(cca, p) - if err != nil { - return err - } - // isSourceAFile defines whether source is a file or not. - // If source is a file and destination is a blob, then destination doesn't needs to be compared against local. - if !isSourceAFile { - err = e.listDestinationAndCompare(cca, p) - if err != nil { - return err - } - } - e.queueSourceFilesForUpload(cca) - - // No Job Part has been dispatched, then dispatch the JobPart. - if e.PartNumber == 0 || - len(e.CopyJobRequest.Transfers) > 0 || - len(e.DeleteJobRequest.Transfers) > 0 { - err = e.dispatchFinalPart(cca) - if err != nil { - return err - } - //cca.waitUntilJobCompletion(true) - cca.setFirstPartOrdered() - } - cca.setScanningComplete() - return nil -} diff --git a/cmd/zt_scenario_helpers_test.go b/cmd/zt_scenario_helpers_test.go new file mode 100644 index 000000000..b3e3f2865 --- /dev/null +++ b/cmd/zt_scenario_helpers_test.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" +) + +const defaultFileSize = 1024 + +type scenarioHelper struct{} + +func (scenarioHelper) generateLocalDirectory(c *chk.C) (dstDirName string) { + dstDirName, err := ioutil.TempDir("", "AzCopySyncDownload") + c.Assert(err, chk.IsNil) + return +} + +// create a test file +func (scenarioHelper) generateFile(filePath string, fileSize int) ([]byte, error) { + // generate random data + _, bigBuff := getRandomDataAndReader(fileSize) + + // create all parent directories + err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + if err != nil { + return nil, err + } + + // write to file and return the data + err = ioutil.WriteFile(filePath, bigBuff, 0666) + return bigBuff, err +} + +func (s scenarioHelper) generateRandomLocalFiles(c *chk.C, dirPath string, numOfFiles int) (fileList []string) { + for i := 0; i < numOfFiles; i++ { + fileName := filepath.Join(dirPath, generateName("random")) + fileList = append(fileList, fileName) + + _, err := s.generateFile(fileName, defaultFileSize) + c.Assert(err, chk.IsNil) + } + return +} + +func (s scenarioHelper) generateFilesFromList(c *chk.C, dirPath string, fileList []string) { + for _, fileName := range fileList { + _, err := s.generateFile(filepath.Join(dirPath, fileName), defaultFileSize) + c.Assert(err, chk.IsNil) + } +} + +// make 30 blobs with random names +// 10 of them at the top level +// 10 of them in sub dir "sub1" +// 10 of them in sub dir "sub2" +func (scenarioHelper) generateCommonRemoteScenario(c *chk.C, containerURL azblob.ContainerURL, prefix string) (blobList []string) { + blobList = make([]string, 30) + for i := 0; i < 10; i++ { + _, blobName1 := createNewBlockBlob(c, containerURL, prefix+"top") + _, blobName2 := createNewBlockBlob(c, containerURL, prefix+"sub1/") + _, blobName3 := createNewBlockBlob(c, containerURL, prefix+"sub2/") + + blobList[3*i] = blobName1 + blobList[3*i+1] = blobName2 + blobList[3*i+2] = blobName3 + } + + return +} + +// create the demanded blobs +func (scenarioHelper) generateBlobs(c *chk.C, containerURL azblob.ContainerURL, blobList []string) { + for _, blobName := range blobList { + blob := containerURL.NewBlockBlobURL(blobName) + cResp, err := blob.Upload(ctx, strings.NewReader(blockBlobDefaultData), azblob.BlobHTTPHeaders{}, + nil, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + c.Assert(cResp.StatusCode(), chk.Equals, 201) + } +} + +// Golang does not have sets, so we have to use a map to fulfill the same functionality +func (scenarioHelper) convertListToMap(list []string) map[string]int { + lookupMap := make(map[string]int) + for _, entryName := range list { + lookupMap[entryName] = 0 + } + + return lookupMap +} + +func (scenarioHelper) getRawContainerURLWithSAS(c *chk.C, containerName string) url.URL { + credential, err := getGenericCredential("") + c.Assert(err, chk.IsNil) + containerURLWithSAS := getContainerURLWithSAS(c, *credential, containerName) + return containerURLWithSAS.URL() +} + +func (scenarioHelper) getRawBlobURLWithSAS(c *chk.C, containerName string, blobName string) url.URL { + credential, err := getGenericCredential("") + c.Assert(err, chk.IsNil) + containerURLWithSAS := getContainerURLWithSAS(c, *credential, containerName) + blobURLWithSAS := containerURLWithSAS.NewBlockBlobURL(blobName) + return blobURLWithSAS.URL() +} diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 5823b91ab..619b54131 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -2,61 +2,126 @@ package cmd import ( "github.com/Azure/azure-storage-azcopy/common" - "github.com/Azure/azure-storage-blob-go/azblob" chk "gopkg.in/check.v1" "io/ioutil" + "path/filepath" "strings" + "time" ) -// create a test file -func generateFile(fileName string, fileSize int) ([]byte, error) { - // generate random data - _, bigBuff := getRandomDataAndReader(fileSize) +const ( + defaultLogVerbosityForSync = "WARNING" + defaultOutputFormatForSync = "text" +) - // write to file and return the data - err := ioutil.WriteFile(fileName, bigBuff, 0666) - return bigBuff, err +func runSyncAndVerify(c *chk.C, raw rawSyncCmdArgs, verifier func(err error)) { + // the simulated user input should parse properly + cooked, err := raw.cook() + c.Assert(err, chk.IsNil) + + // the enumeration ends when process() returns + err = cooked.process() + + // the err is passed to verified, which knows whether it is expected or not + verifier(err) } -// create the necessary blobs with and without virtual directories -func generateCommonScenarioForDownloadSync(c *chk.C) (containerName string, containerUrl azblob.ContainerURL, blobList []string) { - bsu := getBSU() - containerUrl, containerName = createNewContainer(c, bsu) +func validateTransfersAreScheduled(c *chk.C, srcDirName, dstDirName string, expectedTransfers []string, mockedRPC interceptor) { + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(expectedTransfers)) - blobList = make([]string, 30) - for i := 0; i < 10; i++ { - _, blobName1 := createNewBlockBlob(c, containerUrl, "top") - _, blobName2 := createNewBlockBlob(c, containerUrl, "sub1/") - _, blobName3 := createNewBlockBlob(c, containerUrl, "sub2/") + // validate that the right transfers were sent + lookupMap := scenarioHelper{}.convertListToMap(expectedTransfers) + for _, transfer := range mockedRPC.transfers { + srcRelativeFilePath := strings.Replace(transfer.Source, srcDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + dstRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) - blobList[3*i] = blobName1 - blobList[3*i+1] = blobName2 - blobList[3*i+2] = blobName3 - } + // the relative paths should be equal + c.Assert(srcRelativeFilePath, chk.Equals, dstRelativeFilePath) - return + // look up the source from the expected transfers, make sure it exists + _, srcExist := lookupMap[dstRelativeFilePath] + c.Assert(srcExist, chk.Equals, true) + + // look up the destination from the expected transfers, make sure it exists + _, dstExist := lookupMap[dstRelativeFilePath] + c.Assert(dstExist, chk.Equals, true) + } } -// Golang does not have sets, so we have to use a map to fulfill the same functionality -func convertListToMap(list []string) map[string]int { - lookupMap := make(map[string]int) - for _, entryName := range list { - lookupMap[entryName] = 0 +func getDefaultRawInput(src, dst string) rawSyncCmdArgs { + return rawSyncCmdArgs{ + src: src, + dst: dst, + recursive: true, + logVerbosity: defaultLogVerbosityForSync, + output: defaultOutputFormatForSync, + force: true, } +} - return lookupMap +// regular blob->file sync +func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { + bsu := getBSU() + + // set up the container with a single blob + blobName := "singleblobisbest" + blobList := []string{blobName} + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up the destination as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobName + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + + // the file was created after the blob, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // sleep for 1 sec so that the blob's last modified times are guaranteed to be newer + time.Sleep(time.Second) + + // recreate the blob to have a later last modified time + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + mockedRPC.reset() + + // the file was created after the blob, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) } +// regular container->directory sync but destination is empty, so everything has to be transferred func (s *cmdIntegrationSuite) TestSyncDownloadWithEmptyDestination(c *chk.C) { + bsu := getBSU() + // set up the container with numerous blobs - containerName, containerURL, blobList := generateCommonScenarioForDownloadSync(c) + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") defer deleteContainer(c, containerURL) c.Assert(containerURL, chk.NotNil) c.Assert(len(blobList), chk.Not(chk.Equals), 0) // set up the destination with an empty folder - dstDirName, err := ioutil.TempDir("", "AzCopySyncDownload") - c.Assert(err, chk.IsNil) + dstDirName := scenarioHelper{}.generateLocalDirectory(c) // set up interceptor mockedRPC := interceptor{} @@ -64,37 +129,327 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithEmptyDestination(c *chk.C) { mockedRPC.init() // construct the raw input to simulate user input - credential, err := getGenericCredential("") - c.Assert(err, chk.IsNil) - containerUrlWithSAS := getContainerURLWithSAS(c, *credential, containerName) - rawContainerURLWithSAS := containerUrlWithSAS.URL() - raw := rawSyncCmdArgs{ - src: rawContainerURLWithSAS.String(), - dst: dstDirName, - recursive: true, - logVerbosity: "WARNING", - output: "text", - force: false, - } + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) - // the simulated user input should parse properly - cooked, err := raw.cook() - c.Assert(err, chk.IsNil) + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) - // the enumeration ends when process() returns - err = cooked.process() - c.Assert(err, chk.IsNil) + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(blobList)) - // validate that the right number of transfers were scheduled - c.Assert(len(mockedRPC.transfers), chk.Equals, 30) + // validate that the right transfers were sent + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) - // validate that the right transfers were sent - lookupMap := convertListToMap(blobList) - for _, transfer := range mockedRPC.transfers { - localRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + // turn off recursive, this time only top blobs should be transferred + raw.recursive = false + mockedRPC.reset() - // look up the source blob, make sure it matches - _, blobExist := lookupMap[localRelativeFilePath] - c.Assert(blobExist, chk.Equals, true) - } + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + c.Assert(len(mockedRPC.transfers), chk.Not(chk.Equals), len(blobList)) + + for _, transfer := range mockedRPC.transfers { + localRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + c.Assert(strings.Contains(localRelativeFilePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + }) +} + +// regular container->directory sync but destination is identical to the source, transfers are scheduled based on lmt +func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination with a folder that have the exact same files + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // wait for 1 second so that the last modified times of the blobs are guaranteed to be newer + time.Sleep(time.Second) + + // refresh the blobs' last modified time so that they are newer + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) +} + +// regular container->directory sync where destination is missing some files from source, and also has some extra files +func (s *cmdIntegrationSuite) TestSyncDownloadWithMismatchedDestination(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination with a folder that have half of the files from source + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList[0:len(blobList)/2]) + scenarioHelper{}.generateFilesFromList(c, dstDirName, []string{"extraFile1.pdf, extraFile2.txt"}) + expectedOutput := blobList[len(blobList)/2:] // the missing half of source files should be transferred + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, expectedOutput, mockedRPC) + + // make sure the extra files were deleted + currentDstFileList, err := ioutil.ReadDir(dstDirName) + extraFilesFound := false + for _, file := range currentDstFileList { + if strings.Contains(file.Name(), "extra") { + extraFilesFound = true + } + } + + c.Assert(extraFilesFound, chk.Equals, false) + }) +} + +// include flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncDownloadWithIncludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // add special blobs that we wish to include + blobsToInclude := []string{"important.pdf", "includeSub/amazing.jpeg", "exactName"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + raw.include = includeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) + }) +} + +// exclude flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncDownloadWithExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // add special blobs that we wish to exclude + blobsToExclude := []string{"notGood.pdf", "excludeSub/lame.jpeg", "exactName"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToExclude) + excludeString := "*.pdf;*.jpeg;exactName" + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) +} + +// include and exclude flag can work together to limit the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncDownloadWithIncludeAndExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // add special blobs that we wish to include + blobsToInclude := []string{"important.pdf", "includeSub/amazing.jpeg"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // add special blobs that we wish to exclude + // note that the excluded files also match the include string + blobsToExclude := []string{"sorry.pdf", "exclude/notGood.jpeg", "exactName", "sub/exactName"} + scenarioHelper{}.generateBlobs(c, containerURL, blobsToExclude) + excludeString := "so*;not*;exactName" + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + raw.include = includeString + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) + }) +} + +// validate the bug fix for this scenario +func (s *cmdIntegrationSuite) TestSyncDownloadWithMissingDestination(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination as a missing folder + dstDirName := filepath.Join(scenarioHelper{}.generateLocalDirectory(c), "imbatman") + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + // error should not be nil, but the app should not crash either + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} + +// there is a type mismatch between the source and destination +func (s *cmdIntegrationSuite) TestSyncDownloadWithContainerToFile(c *chk.C) { + bsu := getBSU() + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // set up the destination as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobList[0] + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} + +// there is a type mismatch between the source and destination +func (s *cmdIntegrationSuite) TestSyncDownloadWithBlobToDirectory(c *chk.C) { + bsu := getBSU() + + // set up the container with a single blob + blobName := "singleblobisbest" + blobList := []string{blobName} + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up the destination as a directory + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), dstDirName) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) } diff --git a/cmd/zt_sync_filter_test.go b/cmd/zt_sync_filter_test.go new file mode 100644 index 000000000..b077c544d --- /dev/null +++ b/cmd/zt_sync_filter_test.go @@ -0,0 +1,52 @@ +package cmd + +import ( + chk "gopkg.in/check.v1" +) + +type syncFilterSuite struct{} + +var _ = chk.Suite(&syncFilterSuite{}) + +func (s *syncFilterSuite) TestIncludeFilter(c *chk.C) { + // set up the filters + raw := rawSyncCmdArgs{} + includePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") + includeFilter := buildIncludeFilters(includePatternList)[0] + + // test the positive cases + filesToInclude := []string{"bla.pdf", "fancy.jpeg", "socool.jpeg.pdf", "exactName"} + for _, file := range filesToInclude { + passed := includeFilter.pass(genericEntity{name: file}) + c.Assert(passed, chk.Equals, true) + } + + // test the negative cases + notToInclude := []string{"bla.pdff", "fancyjpeg", "socool.jpeg.pdf.wut", "eexactName"} + for _, file := range notToInclude { + passed := includeFilter.pass(genericEntity{name: file}) + c.Assert(passed, chk.Equals, false) + } +} + +func (s *syncFilterSuite) TestExcludeFilter(c *chk.C) { + // set up the filters + raw := rawSyncCmdArgs{} + excludePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") + excludeFilterList := buildExcludeFilters(excludePatternList) + + // test the positive cases + filesToPass := []string{"bla.pdfe", "fancy.jjpeg", "socool.png", "notexactName"} + for _, file := range filesToPass { + dummyProcessor := &dummyProcessor{} + passed := processIfPassedFilters(excludeFilterList, genericEntity{name: file}, dummyProcessor) + c.Assert(passed, chk.Equals, true) + } + + // test the negative cases + filesToNotPass := []string{"bla.pdff", "fancyjpeg", "socool.jpeg.pdf.wut", "eexactName"} + for _, file := range filesToNotPass { + passed := excludeFilter.pass(genericEntity{name: file}) + c.Assert(passed, chk.Equals, false) + } +} diff --git a/cmd/zt_sync_processor_test.go b/cmd/zt_sync_processor_test.go new file mode 100644 index 000000000..6b217963c --- /dev/null +++ b/cmd/zt_sync_processor_test.go @@ -0,0 +1,131 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" + "path/filepath" + "time" +) + +type syncProcessorSuite struct{} + +var _ = chk.Suite(&syncProcessorSuite{}) + +type syncProcessorSuiteHelper struct{} + +// return a list of sample entities +func (syncProcessorSuiteHelper) getSampleEntityList() []genericEntity { + return []genericEntity{ + {name: "file1", relativePath: "file1", lastModifiedTime: time.Now()}, + {name: "file2", relativePath: "file2", lastModifiedTime: time.Now()}, + {name: "file3", relativePath: "sub1/file3", lastModifiedTime: time.Now()}, + {name: "file4", relativePath: "sub1/file4", lastModifiedTime: time.Now()}, + {name: "file5", relativePath: "sub1/sub2/file5", lastModifiedTime: time.Now()}, + {name: "file6", relativePath: "sub1/sub2/file6", lastModifiedTime: time.Now()}, + } +} + +// given a list of entities, return the relative paths in a list, to help with validations +func (syncProcessorSuiteHelper) getExpectedTransferFromEntityList(entityList []genericEntity) []string { + expectedTransfers := make([]string, 0) + for _, entity := range entityList { + expectedTransfers = append(expectedTransfers, entity.relativePath) + } + + return expectedTransfers +} + +func (s *syncProcessorSuite) TestSyncProcessorMultipleFiles(c *chk.C) { + bsu := getBSU() + + // set up source and destination + containerURL, containerName := getContainerURL(c, bsu) + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + cooked, err := raw.cook() + c.Assert(err, chk.IsNil) + + // exercise the sync processor + sampleEntities := syncProcessorSuiteHelper{}.getSampleEntityList() + + for _, numOfParts := range []int{1, 3} { + // note we set the numOfTransfersPerPart here + syncProcessor := newSyncTransferProcessor(&cooked, len(sampleEntities)/numOfParts) + + // go through the entities and make sure they are processed without error + for _, entity := range sampleEntities { + err := syncProcessor.process(entity) + c.Assert(err, chk.IsNil) + } + + // make sure everything has been dispatched apart from the final one + c.Assert(syncProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(numOfParts-1)) + + // dispatch final part + jobInitiated, err := syncProcessor.dispatchFinalPart() + c.Assert(jobInitiated, chk.Equals, true) + c.Assert(err, chk.IsNil) + + // assert the right transfers were scheduled + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, + syncProcessorSuiteHelper{}.getExpectedTransferFromEntityList(sampleEntities), mockedRPC) + + mockedRPC.reset() + } +} + +func (s *syncProcessorSuite) TestSyncProcessorSingleFile(c *chk.C) { + bsu := getBSU() + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up the container with a single blob + blobList := []string{"singlefile101"} + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + c.Assert(containerURL, chk.NotNil) + + // set up the directory with a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobList[0] + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + cooked, err := raw.cook() + c.Assert(err, chk.IsNil) + + // exercise the sync processor + syncProcessor := newSyncTransferProcessor(&cooked, 2) + entity := genericEntity{ + name: blobList[0], + relativePath: "", + lastModifiedTime: time.Now(), + } + err = syncProcessor.process(entity) + c.Assert(err, chk.IsNil) + + // no part should have been dispatched + c.Assert(syncProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(0)) + + // dispatch final part + jobInitiated, err := syncProcessor.dispatchFinalPart() + c.Assert(jobInitiated, chk.Equals, true) + + // assert the right transfers were scheduled + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, + blobList, mockedRPC) +} diff --git a/cmd/zt_sync_traverser_test.go b/cmd/zt_sync_traverser_test.go new file mode 100644 index 000000000..7cd7528df --- /dev/null +++ b/cmd/zt_sync_traverser_test.go @@ -0,0 +1,181 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" + "path" + "path/filepath" + "strings" +) + +type syncTraverserSuite struct{} + +var _ = chk.Suite(&syncTraverserSuite{}) + +type dummyProcessor struct { + record []genericEntity +} + +func (d *dummyProcessor) process(entity genericEntity) (err error) { + d.record = append(d.record, entity) + return +} + +func (s *syncTraverserSuite) TestSyncTraverserSingleEntity(c *chk.C) { + bsu := getBSU() + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // test two scenarios, either blob is at the root virtual dir, or inside sub virtual dirs + for _, blobName := range []string{"sub1/sub2/singleblobisbest", "nosubsingleblob"} { + // set up the container with a single blob + blobList := []string{blobName} + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + c.Assert(containerURL, chk.NotNil) + + // set up the directory as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobName + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // simulate cca with typical values + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + cca, err := raw.cook() + c.Assert(err, chk.IsNil) + + // construct a local traverser + localTraverser := newLocalTraverser(&cca, false) + + // invoke the local traversal with a dummy processor + localDummyProcessor := dummyProcessor{} + err = localTraverser.traverse(&localDummyProcessor, nil) + c.Assert(err, chk.IsNil) + c.Assert(len(localDummyProcessor.record), chk.Equals, 1) + + // construct a blob traverser + blobTraverser, err := newBlobTraverser(&cca, true) + c.Assert(err, chk.IsNil) + + // invoke the local traversal with a dummy processor + blobDummyProcessor := dummyProcessor{} + err = blobTraverser.traverse(&blobDummyProcessor, nil) + c.Assert(err, chk.IsNil) + c.Assert(len(blobDummyProcessor.record), chk.Equals, 1) + + // assert the important info are correct + c.Assert(localDummyProcessor.record[0].name, chk.Equals, blobDummyProcessor.record[0].name) + c.Assert(localDummyProcessor.record[0].relativePath, chk.Equals, blobDummyProcessor.record[0].relativePath) + } +} + +func (s *syncTraverserSuite) TestSyncTraverserContainerAndLocalDirectory(c *chk.C) { + bsu := getBSU() + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up the container with numerous blobs + fileList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, "") + c.Assert(containerURL, chk.NotNil) + + // set up the destination with a folder that have the exact same files + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, fileList) + + // test two scenarios, either recursive or not + for _, isRecursiveOn := range []bool{true, false} { + // simulate cca with typical values + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + raw.recursive = isRecursiveOn + cca, err := raw.cook() + c.Assert(err, chk.IsNil) + + // construct a local traverser + localTraverser := newLocalTraverser(&cca, false) + + // invoke the local traversal with an indexer + localIndexer := newDestinationIndexer() + err = localTraverser.traverse(localIndexer, nil) + c.Assert(err, chk.IsNil) + + // construct a blob traverser + blobTraverser, err := newBlobTraverser(&cca, true) + c.Assert(err, chk.IsNil) + + // invoke the local traversal with a dummy processor + blobDummyProcessor := dummyProcessor{} + err = blobTraverser.traverse(&blobDummyProcessor, nil) + c.Assert(err, chk.IsNil) + + // make sure the results are the same + c.Assert(len(blobDummyProcessor.record), chk.Equals, len(localIndexer.indexMap)) + for _, entity := range blobDummyProcessor.record { + correspondingLocalFile, present := localIndexer.indexMap[entity.relativePath] + + c.Assert(present, chk.Equals, true) + c.Assert(correspondingLocalFile.name, chk.Equals, entity.name) + c.Assert(correspondingLocalFile.isMoreRecentThan(entity), chk.Equals, true) + + if !isRecursiveOn { + c.Assert(strings.Contains(entity.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + } + } +} + +func (s *syncTraverserSuite) TestSyncTraverserVirtualAndLocalDirectory(c *chk.C) { + bsu := getBSU() + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up the container with numerous blobs + virDirName := "virdir" + fileList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, virDirName+"/") + c.Assert(containerURL, chk.NotNil) + + // set up the destination with a folder that have the exact same files + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateFilesFromList(c, dstDirName, fileList) + + // test two scenarios, either recursive or not + for _, isRecursiveOn := range []bool{true, false} { + // simulate cca with typical values + rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) + raw := getDefaultRawInput(rawVirDirURLWithSAS.String(), path.Join(dstDirName, virDirName)) + raw.recursive = isRecursiveOn + cca, err := raw.cook() + c.Assert(err, chk.IsNil) + + // construct a local traverser + localTraverser := newLocalTraverser(&cca, false) + + // invoke the local traversal with an indexer + localIndexer := newDestinationIndexer() + err = localTraverser.traverse(localIndexer, nil) + c.Assert(err, chk.IsNil) + + // construct a blob traverser + blobTraverser, err := newBlobTraverser(&cca, true) + c.Assert(err, chk.IsNil) + + // invoke the local traversal with a dummy processor + blobDummyProcessor := dummyProcessor{} + err = blobTraverser.traverse(&blobDummyProcessor, nil) + c.Assert(err, chk.IsNil) + + // make sure the results are the same + c.Assert(len(blobDummyProcessor.record), chk.Equals, len(localIndexer.indexMap)) + for _, entity := range blobDummyProcessor.record { + correspondingLocalFile, present := localIndexer.indexMap[entity.relativePath] + + c.Assert(present, chk.Equals, true) + c.Assert(correspondingLocalFile.name, chk.Equals, entity.name) + c.Assert(correspondingLocalFile.isMoreRecentThan(entity), chk.Equals, true) + + if !isRecursiveOn { + c.Assert(strings.Contains(entity.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + } + } +} diff --git a/cmd/zt_test_interceptor.go b/cmd/zt_test_interceptor.go index f254cf55a..27a30b96c 100644 --- a/cmd/zt_test_interceptor.go +++ b/cmd/zt_test_interceptor.go @@ -53,6 +53,11 @@ func (i *interceptor) init() { glcm = mockedLifecycleManager{} } +func (i *interceptor) reset() { + i.transfers = make([]common.CopyTransfer, 0) + i.lastRequest = nil +} + // this lifecycle manager substitute does not perform any action type mockedLifecycleManager struct{} diff --git a/common/rpc-models.go b/common/rpc-models.go index 931cc498d..75b96d927 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -74,8 +74,8 @@ type SyncJobPartOrderRequest struct { FromTo FromTo PartNumber PartNumber LogLevel LogLevel - Include map[string]int - Exclude map[string]int + Include []string + Exclude []string BlockSizeInBytes uint32 SourceSAS string DestinationSAS string @@ -89,7 +89,7 @@ type SyncJobPartOrderRequest struct { CommandString string CredentialInfo CredentialInfo - SourceFiles map[string]time.Time + LocalFiles map[string]time.Time SourceFilesToExclude map[string]time.Time } diff --git a/go.mod b/go.mod index 503c255c6..132abbbd0 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/Azure/azure-storage-azcopy require ( github.com/Azure/azure-pipeline-go v0.1.8 - github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c + github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 github.com/Azure/azure-storage-file-go v0.0.0-20190108093629-d93e19c84c2a github.com/Azure/go-autorest v10.15.2+incompatible github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda diff --git a/go.sum b/go.sum index 18919151e..156ae7236 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/Azure/azure-pipeline-go v0.1.8 h1:KmVRa8oFMaargVesEuuEoiLCQ4zCCwQ8QX/ github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c h1:Y5ueznoCekgCWBytF1Q9lTpZ3tJeX37dQtCcGjMCLYI= github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= +github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 h1:QjGHsWFbJyl312t0BtgkmZy2TTYA++FF0UakGbr3ZhQ= +github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/azure-storage-file-go v0.0.0-20180929015327-d6c64f9676be h1:2TBD/QJYxkQf2jAHk/radyLgEfUpV8eqvRPvnjJt9EA= github.com/Azure/azure-storage-file-go v0.0.0-20180929015327-d6c64f9676be/go.mod h1:N5mXnKL8ZzcrxcNfqrcfWhiaCPAGagfTxH0/IwPN/LI= github.com/Azure/azure-storage-file-go v0.0.0-20190108093629-d93e19c84c2a h1:5OfEqciJHSMkxAWgJP1b3JTmzWNRlq9L9IgOxPNlBOM= diff --git a/testSuite/scripts/test_blob_download.py b/testSuite/scripts/test_blob_download.py index af40756d4..d46311b89 100644 --- a/testSuite/scripts/test_blob_download.py +++ b/testSuite/scripts/test_blob_download.py @@ -152,137 +152,6 @@ def test_blob_download_with_special_characters(self): result = util.Command("testBlob").add_arguments(filepath).add_arguments(resource_url).execute_azcopy_verify() self.assertTrue(result) - def test_sync_blob_download_without_wildcards(self): - # created a directory and created 10 files inside the directory - dir_name = "sync_download_without_wildcards" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # upload the directory - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(dir_sas).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # execute the validator and verify the downloaded dir - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # sync the source and destination - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - - def test_sync_blob_download_with_wildcards(self): - # created a directory and created 10 files inside the directory - dir_name = "sync_download_with_wildcards" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # upload the directory - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(dir_sas).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # execute the validator and verify the downloaded dir - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # add "*" at the end of dir sas - # since both the source and destination are in sync, it will fail - dir_sas = util.append_text_path_resource_sas(dir_sas, "*") - # sync the source and destination - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - subdir1 = os.path.join(dir_name, "subdir1") - subdir1_file_path = util.create_test_n_files(1024, 10, subdir1) - - subdir2 = os.path.join(dir_name, "subdir2") - subdir2_file_path = util.create_test_n_files(1024, 10, subdir2) - - # upload the directory - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # Download the directory to match the blob modified time - result = util.Command("copy").add_arguments(dir_sas).add_arguments(util.test_directory_path). \ - add_flags("log-level", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # sync the source and destination - # add extra wildcards - # since source and destination both are in sync, it will will not perform any sync.s - dir_sas = util.append_text_path_resource_sas(dir_sas, "*/*.txt") - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # delete 5 files inside each sub-directories locally - for r in range(5, 10): - filename = "test101024_" + str(r) + ".txt" - filepath = os.path.join(subdir1_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file ' + filepath) - filepath = os.path.join(subdir2_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file ' + filepath) - # 10 files have been deleted inside the sub-dir - # sync remote to local - # 10 files will be downloaded - result = util.Command("sync").add_arguments(dir_sas).add_arguments(dir_n_files_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output","json").\ - execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - # Number of Expected Transfer should be 10 since 10 files were deleted - self.assertEquals(x.CopyTransfersCompleted, 10) - self.assertEquals(x.CopyTransfersFailed, 0) - # test_download_1kb_blob verifies the download of 1Kb blob using azcopy. def test_download_1kb_blob_with_oauth(self): self.util_test_download_1kb_blob_with_oauth() diff --git a/testSuite/scripts/test_upload_block_blob.py b/testSuite/scripts/test_upload_block_blob.py index 330eff9f6..a91752df1 100644 --- a/testSuite/scripts/test_upload_block_blob.py +++ b/testSuite/scripts/test_upload_block_blob.py @@ -580,191 +580,6 @@ def test_download_blob_exclude_flag(self): self.assertEquals(x.TransfersCompleted, 10) self.assertEquals(x.TransfersFailed, 0) - def test_sync_local_to_blob_without_wildCards(self): - # create 10 files inside the dir 'sync_local_blob' - dir_name = "sync_local_blob" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # create sub-dir inside dir sync_local_blob - # create 10 files inside the sub-dir of size 1024 - sub_dir_name = os.path.join(dir_name, "sub_dir_sync_local_blob") - sub_dir_n_file_path = util.create_test_n_files(1024, 10, sub_dir_name) - - # uploading the directory with 20 files in it. - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - # execute the validator and validating the uploaded directory. - destination = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # execute a sync command - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - # since source and destination both are in sync, there should no sync and the azcopy should exit with error code - self.assertTrue(result) - try: - shutil.rmtree(sub_dir_n_file_path) - except: - self.fail('error deleting the directory' + sub_dir_n_file_path) - - # deleted entire sub-dir inside the dir created above - # sync between source and destination should delete the sub-dir on container - # number of successful transfer should be equal to 10 - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - - # Number of Expected Transfer should be 10 since sub-dir is to exclude which has 10 files in it. - self.assertEquals(x.DeleteTransfersCompleted, 10) - self.assertEquals(x.DeleteTransfersFailed, 0) - - # delete 5 files inside the directory - for r in range(5, 10): - filename = "test101024_" + str(r) + ".txt" - filepath = os.path.join(dir_n_files_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file ' + filepath) - - # sync between source and destination should delete the deleted files on container - # number of successful transfer should be equal to 5 - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - - # Number of Expected Transfer should be 10 since 10 files were deleted - self.assertEquals(x.DeleteTransfersCompleted, 5) - self.assertEquals(x.DeleteTransfersFailed, 0) - - # change the modified time of file - # perform the sync - # expected number of transfer is 1 - filepath = os.path.join(dir_n_files_path, "test101024_0.txt") - st = os.stat(filepath) - atime = st[ST_ATIME] # access time - mtime = st[ST_MTIME] # modification time - new_mtime = mtime + (4 * 3600) # new modification time - os.utime(filepath, (atime, new_mtime)) - # sync source to destination - result = util.Command("sync").add_arguments(dir_n_files_path).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - # Number of Expected Transfer should be 1 since 1 file's modified time was changed - self.assertEquals(x.CopyTransfersCompleted, 1) - self.assertEquals(x.CopyTransfersFailed, 0) - - def test_sync_local_to_blob_with_wildCards(self): - # create 10 files inside the dir 'sync_local_blob' - dir_name = "sync_local_blob_wc" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # create sub-dir inside dir sync_local_blob_wc - # create 10 files inside the sub-dir of size 1024 - sub_dir_1 = os.path.join(dir_name, "sub_dir_1") - sub_dir1_n_file_path = util.create_test_n_files(1024, 10, sub_dir_1) - - # create sub-dir inside dir sync_local_blob_wc - sub_dir_2 = os.path.join(dir_name, "sub_dir_2") - sub_dir2_n_file_path = util.create_test_n_files(1024, 10, sub_dir_2) - - # uploading the directory with 30 files in it. - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator and validating the uploaded directory. - destination = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - # download the destination to the source to match the last modified time - result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("output", "json"). \ - add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command_get_output() - self.assertNotEquals(result, None) - - # add wildcard at the end of dirpath - dir_n_files_path_wcard = os.path.join(dir_n_files_path, "*") - # execute a sync command - dir_sas = util.get_resource_sas(dir_name) - result = util.Command("sync").add_arguments(dir_n_files_path_wcard).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - # since source and destination both are in sync, there should no sync and the azcopy should exit with error code - self.assertTrue(result) - - # sync all the files the ends with .txt extension inside all sub-dirs inside inside - # sd_dir_n_files_path_wcard is in format dir/*/*.txt - sd_dir_n_files_path_wcard = os.path.join(dir_n_files_path_wcard, "*.txt") - result = util.Command("sync").add_arguments(sd_dir_n_files_path_wcard).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").execute_azcopy_copy_command() - # since source and destination both are in sync, there should no sync and the azcopy should exit with error code - self.assertTrue(result) - - # remove 5 files inside both the sub-directories - for r in range(5, 10): - filename = "test101024_" + str(r) + ".txt" - filepath = os.path.join(sub_dir1_n_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file '+ filepath) - filepath = os.path.join(sub_dir2_n_file_path, filename) - try: - os.remove(filepath) - except: - self.fail('error deleting the file '+ filepath) - # sync all the files the ends with .txt extension inside all sub-dirs inside inside - # since 5 files inside each sub-dir are deleted, sync will have total 10 transfer - # 10 files will deleted from container - sd_dir_n_files_path_wcard = os.path.join(dir_n_files_path_wcard, "*.txt") - result = util.Command("sync").add_arguments(sd_dir_n_files_path_wcard).add_arguments(dir_sas). \ - add_flags("log-level", "info").add_flags("recursive", "true").add_flags("force", "true").add_flags("output", - "json").execute_azcopy_copy_command_get_output() - # parse the result to get the last job progress summary - result = util.parseAzcopyOutput(result) - try: - # parse the Json Output - x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) - except: - self.fail('error parsing the output in Json Format') - # Number of Expected Transfer should be 10 since 10 files were deleted - self.assertEquals(x.DeleteTransfersCompleted, 10) - self.assertEquals(x.DeleteTransfersFailed, 0) - - def test_0KB_blob_upload(self): # Creating a single File Of size 0 KB filename = "test0KB.txt" From 2e91b6f3bfa808f29cf8e369728e870e301c52fd Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Wed, 6 Feb 2019 18:26:15 -0800 Subject: [PATCH 13/64] Refactored generic code and put into different files --- cmd/helpMessages.go | 25 ++ cmd/sync.go | 87 +++-- cmd/syncEnumerator.go | 266 ++++---------- cmd/syncFilter.go | 96 ++--- cmd/syncIndexer.go | 33 ++ cmd/syncProcessor.go | 218 ++++++------ cmd/syncTraverser.go | 255 ++------------ cmd/zc_enumerator.go | 159 +++++++++ cmd/zc_filter.go | 80 +++++ cmd/zc_processor.go | 101 ++++++ cmd/zc_traverser_blob.go | 115 ++++++ cmd/zc_traverser_local.go | 101 ++++++ cmd/zt_generic_filter_test.go | 55 +++ cmd/zt_generic_processor_test.go | 122 +++++++ ...r_test.go => zt_generic_traverser_test.go} | 113 +++--- ...rceptor.go => zt_interceptors_for_test.go} | 9 + ...est.go => zt_scenario_helpers_for_test.go} | 31 +- cmd/zt_sync_download_test.go | 27 +- cmd/zt_sync_filter_test.go | 103 ++++-- cmd/zt_sync_processor_test.go | 146 +++----- cmd/zt_sync_upload_test.go | 333 ++++++++++++++++++ common/rpc-models.go | 25 -- 22 files changed, 1658 insertions(+), 842 deletions(-) create mode 100644 cmd/syncIndexer.go create mode 100644 cmd/zc_enumerator.go create mode 100644 cmd/zc_filter.go create mode 100644 cmd/zc_processor.go create mode 100644 cmd/zc_traverser_blob.go create mode 100644 cmd/zc_traverser_local.go create mode 100644 cmd/zt_generic_filter_test.go create mode 100644 cmd/zt_generic_processor_test.go rename cmd/{zt_sync_traverser_test.go => zt_generic_traverser_test.go} (52%) rename cmd/{zt_test_interceptor.go => zt_interceptors_for_test.go} (93%) rename cmd/{zt_scenario_helpers_test.go => zt_scenario_helpers_for_test.go} (78%) create mode 100644 cmd/zt_sync_upload_test.go diff --git a/cmd/helpMessages.go b/cmd/helpMessages.go index ec796f776..b2e240281 100644 --- a/cmd/helpMessages.go +++ b/cmd/helpMessages.go @@ -178,6 +178,12 @@ const syncCmdLongDescription = ` Replicates source to the destination location. The last modified times are used for comparison. The supported pairs are: - local <-> Azure Blob (SAS or OAuth authentication) +Please note that the sync command differs from the copy command in several ways: + 0. The recursive flag is on by default. + 1. The source and destination should not contain patterns(such as * or ?). + 2. The include/exclude flags can be a list of patterns matching to the file names. Please refer to the example section for illustration. + 3. If there are files/blobs at the destination that are not present at the source, the user will be prompted to delete them. This prompt can be silenced by using the corresponding flags to automatically answer the deletion question. + Advanced: Please note that AzCopy automatically detects the Content-Type of files when uploading from local disk, based on file extension or file content(if no extension). @@ -188,3 +194,22 @@ The built-in lookup table is small but on unix it is augmented by the local syst On Windows, MIME types are extracted from the registry. ` + +const syncCmdExample = ` +Sync a single file: + - azcopy sync "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" + +Sync an entire directory including its sub-directories (note that recursive is by default on): + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" + +Sync only the top files inside a directory but not its sub-directories: + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=false + +Sync a subset of files in a directory (ex: only jpg and pdf files, or if the file name is "exactName"): + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --include="*.jpg;*.pdf;exactName" + +Sync an entire directory but exclude certain files from the scope (ex: every file that starts with foo or ends with bar): + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --exclude="foo*;*bar" + +Note: if include/exclude flags are used together, only files matching the include patterns would be looked at, but those matching the exclude patterns would be always be ignored. +` diff --git a/cmd/sync.go b/cmd/sync.go index de0e230bc..2a7359c32 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -23,7 +23,6 @@ package cmd import ( "context" "encoding/json" - "errors" "fmt" "time" @@ -39,7 +38,7 @@ import ( "github.com/spf13/cobra" ) -// TODO +// TODO plug this in // a max is set because we cannot buffer infinite amount of destination file info in memory const MaxNumberOfFilesAllowedInSync = 10000000 @@ -192,6 +191,7 @@ type cookedSyncCmdArgs struct { // this flag is set by the enumerator // it is useful to indicate whether we are simply waiting for the purpose of cancelling + // this is set to true once the final part has been dispatched isEnumerationComplete bool // defines the scanning status of the sync operation. @@ -273,19 +273,42 @@ func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { } func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { - if !cca.scanningComplete() { - lcm.Progress(fmt.Sprintf("%v File Scanned at Source, %v Files Scanned at Destination", - atomic.LoadUint64(&cca.atomicSourceFilesScanned), atomic.LoadUint64(&cca.atomicDestinationFilesScanned))) - return + var summary common.ListSyncJobSummaryResponse + var throughput float64 + var jobDone bool + + // fetch a job status and compute throughput if the first part was dispatched + if cca.firstPartOrdered() { + Rpc(common.ERpcCmd.ListSyncJobSummary(), &cca.jobID, &summary) + jobDone = summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() + + // compute the average throughput for the last time interval + bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) * 8 / float64(1024*1024)) + timeElapsed := time.Since(cca.intervalStartTime).Seconds() + throughput = common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) + + // reset the interval timer and byte count + cca.intervalStartTime = time.Now() + cca.intervalBytesTransferred = summary.BytesOverWire } - // If the first part isn't ordered yet, no need to fetch the progress summary. - if !cca.firstPartOrdered() { + + // first part not dispatched, and we are still scanning + // so a special message is outputted to notice the user that we are not stalling + if !jobDone && !cca.scanningComplete() { + // skip the interactive message if we were triggered by another tool + if cca.output == common.EOutputFormat.Json() { + return + } + + var throughputString string + if cca.firstPartOrdered() { + throughputString = fmt.Sprintf(", 2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) + } + + lcm.Progress(fmt.Sprintf("%v File Scanned at Source, %v Files Scanned at Destination%s", + atomic.LoadUint64(&cca.atomicSourceFilesScanned), atomic.LoadUint64(&cca.atomicDestinationFilesScanned), throughputString)) return } - // fetch a job status - var summary common.ListSyncJobSummaryResponse - Rpc(common.ERpcCmd.ListSyncJobSummary(), &cca.jobID, &summary) - jobDone := summary.JobStatus.IsJobDone() // if json output is desired, simply marshal and return // note that if job is already done, we simply exit @@ -326,27 +349,11 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { summary.JobStatus), exitCode) } - // if json is not needed, and job is not done, then we generate a message that goes nicely on the same line - // display a scanning keyword if the job is not completely ordered - var scanningString = "" - if !summary.CompleteJobOrdered { - scanningString = "(scanning...)" - } - - // compute the average throughput for the last time interval - bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) * 8 / float64(1024*1024)) - timeElapsed := time.Since(cca.intervalStartTime).Seconds() - throughPut := common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) - - // reset the interval timer and byte count - cca.intervalStartTime = time.Now() - cca.intervalBytesTransferred = summary.BytesOverWire - - lcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total%s, 2-sec Throughput (Mb/s): %v", + lcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total, 2-sec Throughput (Mb/s): %v", summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted, summary.CopyTransfersFailed+summary.DeleteTransfersFailed, summary.CopyTotalTransfers+summary.DeleteTotalTransfers-(summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted+summary.CopyTransfersFailed+summary.DeleteTransfersFailed), - summary.CopyTotalTransfers+summary.DeleteTotalTransfers, scanningString, ste.ToFixed(throughPut, 4))) + summary.CopyTotalTransfers+summary.DeleteTotalTransfers, ste.ToFixed(throughput, 4))) } func (cca *cookedSyncCmdArgs) process() (err error) { @@ -386,7 +393,10 @@ func (cca *cookedSyncCmdArgs) process() (err error) { switch cca.fromTo { case common.EFromTo.LocalBlob(): - return errors.New("work in progress") + enumerator, err = newSyncUploadEnumerator(cca) + if err != nil { + return err + } case common.EFromTo.BlobLocal(): enumerator, err = newSyncDownloadEnumerator(cca) if err != nil { @@ -396,9 +406,13 @@ func (cca *cookedSyncCmdArgs) process() (err error) { return fmt.Errorf("the given source/destination pair is currently not supported") } + // trigger the progress reporting + cca.waitUntilJobCompletion(false) + + // trigger the enumeration err = enumerator.enumerate() if err != nil { - return fmt.Errorf("error starting the sync between source %s and destination %s. Failed with error %s", cca.source, cca.destination, err.Error()) + return err } return nil } @@ -411,6 +425,7 @@ func init() { Aliases: []string{"sc", "s"}, Short: syncCmdShortDescription, Long: syncCmdLongDescription, + Example: syncCmdExample, Args: func(cmd *cobra.Command, args []string) error { if len(args) != 2 { return fmt.Errorf("2 arguments source and destination are required for this command. Number of commands passed %d", len(args)) @@ -427,7 +442,7 @@ func init() { cooked.commandString = copyHandlerUtil{}.ConstructCommandStringFromArgs() err = cooked.process() if err != nil { - glcm.Exit("error performing the sync between source and destination. Failed with error "+err.Error(), common.EExitCode.Error()) + glcm.Exit("Cannot perform sync due to error: "+err.Error(), common.EExitCode.Error()) } glcm.SurrenderControl() @@ -435,11 +450,10 @@ func init() { } rootCmd.AddCommand(syncCmd) - syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when syncing between directories.") + syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", true, "true by default, look into sub-directories recursively when syncing between directories.") syncCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "use this block(chunk) size when uploading/downloading to/from Azure Storage.") syncCmd.PersistentFlags().StringVar(&raw.include, "include", "", "only include files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") - syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "follow symbolic links when performing sync from local file system.") syncCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "WARNING", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") syncCmd.PersistentFlags().BoolVar(&raw.force, "force", false, "defines user's decision to delete extra files at the destination that are not present at the source. "+ @@ -447,5 +461,8 @@ func init() { syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading.") // TODO: should the previous line list the allowable values? + // TODO follow sym link is not implemented, clarify behavior first + //syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "follow symbolic links when performing sync from local file system.") + // TODO sync does not support any BlobAttributes, this functionality should be added } diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 0a668927f..d5b62a1cf 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -2,241 +2,131 @@ package cmd import ( "errors" + "fmt" "github.com/Azure/azure-storage-azcopy/common" - "strings" - "time" ) -// -------------------------------------- Component Definitions -------------------------------------- \\ -// the following interfaces and structs allow the sync enumerator -// to be generic and has as little duplicated code as possible - -// represent a local or remote resource entity (ex: local file, blob, etc.) -// we can add more properties if needed, as this is easily extensible -type genericEntity struct { - name string - lastModifiedTime time.Time - size int64 - - // partial path relative to its directory - // example: dir=/var/a/b/c fullPath=/var/a/b/c/d/e/f.pdf relativePath=d/e/f.pdf - relativePath string -} - -func (entity *genericEntity) isMoreRecentThan(entity2 genericEntity) bool { - return entity.lastModifiedTime.After(entity2.lastModifiedTime) -} - -// capable of traversing a resource, pass each entity to the given entityProcessor if it passes all the filters -type resourceTraverser interface { - traverse(processor entityProcessor, filters []entityFilter) error -} - -// given a genericEntity, process it accordingly -type entityProcessor interface { - process(entity genericEntity) error -} - -// given a genericEntity, verify if it satisfies the defined conditions -type entityFilter interface { - pass(entity genericEntity) bool -} - -// -------------------------------------- Generic Enumerator -------------------------------------- \\ - -type syncEnumerator struct { - // these allow us to go through the source and destination - sourceTraverser resourceTraverser - destinationTraverser resourceTraverser - - // filters apply to both the source and destination - filters []entityFilter - - // the processor responsible for scheduling copy transfers - copyTransferScheduler entityProcessor - - // the processor responsible for scheduling delete transfers - deleteTransferScheduler entityProcessor - - // a finalizer that is always called if the enumeration finishes properly - finalize func() error -} - -func (e *syncEnumerator) enumerate() (err error) { - destinationIndexer := newDestinationIndexer() - - // enumerate the destination and build lookup map - err = e.destinationTraverser.traverse(destinationIndexer, e.filters) - if err != nil { - return - } - - // add the destinationIndexer as an extra filter to the list - e.filters = append(e.filters, destinationIndexer) +// -------------------------------------- Implemented Enumerators -------------------------------------- \\ - // enumerate the source and schedule transfers - err = e.sourceTraverser.traverse(e.copyTransferScheduler, e.filters) +// download implies transferring from a remote resource to the local disk +// in this scenario, the destination is scanned/indexed first +// then the source is scanned and filtered based on what the destination contains +func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { + destinationTraverser, err := newLocalTraverserForSync(cca, false) if err != nil { - return + return nil, err } - // delete extra files at the destination if needed - err = destinationIndexer.traverse(e.deleteTransferScheduler, nil) + sourceTraverser, err := newBlobTraverserForSync(cca, true) if err != nil { - return + return nil, err } - // execute the finalize func which may perform useful clean up steps - err = e.finalize() - if err != nil { - return + // verify that the traversers are targeting the same type of resources + _, isSingleBlob := sourceTraverser.getPropertiesIfSingleBlob() + _, isSingleFile, _ := destinationTraverser.getInfoIfSingleFile() + if isSingleBlob != isSingleFile { + return nil, errors.New("sync must happen between source and destination of the same type: either blob <-> file, or container/virtual directory <-> local directory") } - return -} - -// the destinationIndexer implements both entityProcessor, entityFilter, and resourceTraverser -// it is essential for the generic enumerator to work -// it can: -// 1. accumulate a lookup map with given destination entities -// 2. serve as a filter to check whether a given entity is the lookup map -// 3. go through the entities in the map like a traverser -type destinationIndexer struct { - indexMap map[string]genericEntity -} - -func newDestinationIndexer() *destinationIndexer { - indexer := destinationIndexer{} - indexer.indexMap = make(map[string]genericEntity) - - return &indexer -} - -func (i *destinationIndexer) process(entity genericEntity) (err error) { - i.indexMap[entity.relativePath] = entity - return -} - -// it will only pass items that are: -// 1. not present in the map -// 2. present but is more recent than the entry in the map -// note: we remove the entity if it is present -func (i *destinationIndexer) pass(entity genericEntity) bool { - entityInMap, present := i.indexMap[entity.relativePath] + transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart) + includeFilters := buildIncludeFilters(cca.include) + excludeFilters := buildExcludeFilters(cca.exclude) - // if the given entity is more recent, we let it pass - if present { - defer delete(i.indexMap, entity.relativePath) + // set up the filters in the right order + filters := append(includeFilters, excludeFilters...) - if entity.isMoreRecentThan(entityInMap) { - return true - } - } else if !present { - return true - } + // set up the comparator so that the source/destination can be compared + indexer := newObjectIndexer() + comparator := newSyncSourceFilter(indexer) - return false -} - -// go through the entities in the map to process them -func (i *destinationIndexer) traverse(processor entityProcessor, filters []entityFilter) (err error) { - for _, value := range i.indexMap { - if !passedFilters(filters, value) { - continue + finalize := func() error { + jobInitiated, err := transferScheduler.dispatchFinalPart() + if err != nil { + return err } - err = processor.process(value) + // remove the extra files at the destination that were not present at the source + deleteScheduler := newSyncLocalDeleteProcessor(cca) + err = indexer.traverse(deleteScheduler.removeImmediately, nil) if err != nil { - return + return err } - } - return -} -// -------------------------------------- Helper Funcs -------------------------------------- \\ - -func passedFilters(filters []entityFilter, entity genericEntity) bool { - if filters != nil && len(filters) > 0 { - // loop through the filters, if any of them fail, then return false - for _, filter := range filters { - if !filter.pass(entity) { - return false - } + if !jobInitiated && !deleteScheduler.wasAnyFileDeleted() { + return errors.New("the source and destination are already in sync") + } else if !jobInitiated && deleteScheduler.wasAnyFileDeleted() { + // some files were deleted but no transfer scheduled + glcm.Exit("the source and destination are now in sync", common.EExitCode.Success()) } - } - - return true -} -func processIfPassedFilters(filters []entityFilter, entity genericEntity, processor entityProcessor) (err error) { - if passedFilters(filters, entity) { - err = processor.process(entity) + cca.setScanningComplete() + return nil } - return + return newSyncEnumerator(destinationTraverser, sourceTraverser, indexer, filters, comparator, + transferScheduler.scheduleCopyTransfer, finalize), nil } -// entity names are useful for filters -func getEntityNameOnly(fullPath string) (nameOnly string) { - lastPathSeparator := strings.LastIndex(fullPath, common.AZCOPY_PATH_SEPARATOR_STRING) - - // if there is a path separator and it is not the last character - if lastPathSeparator > 0 && lastPathSeparator != len(fullPath)-1 { - // then we separate out the name of the entity - nameOnly = fullPath[lastPathSeparator+1:] - } else { - nameOnly = fullPath +// upload implies transferring from a local disk to a remote resource +// in this scenario, the local disk (source) is scanned/indexed first +// then the destination is scanned and filtered based on what the destination contains +func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { + sourceTraverser, err := newLocalTraverserForSync(cca, true) + if err != nil { + return nil, err } - return -} - -// -------------------------------------- Implemented Enumerators -------------------------------------- \\ - -func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { - sourceTraverser, err := newBlobTraverser(cca, true) + destinationTraverser, err := newBlobTraverserForSync(cca, false) if err != nil { - // this is unexpected - // if there is an error here, the URL was probably not valid return nil, err } - destinationTraverser := newLocalTraverser(cca, false) - deleteScheduler := newSyncLocalDeleteProcessor(cca, false) - transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart) - - _, isSingleBlob := sourceTraverser.getPropertiesIfSingleBlob() - _, isSingleFile, _ := destinationTraverser.getInfoIfSingleFile() + // verify that the traversers are targeting the same type of resources + _, isSingleBlob := destinationTraverser.getPropertiesIfSingleBlob() + _, isSingleFile, _ := sourceTraverser.getInfoIfSingleFile() if isSingleBlob != isSingleFile { return nil, errors.New("sync must happen between source and destination of the same type: either blob <-> file, or container/virtual directory <-> local directory") } + transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart) + includeFilters := buildIncludeFilters(cca.include) + excludeFilters := buildExcludeFilters(cca.exclude) + + // set up the filters in the right order + filters := append(includeFilters, excludeFilters...) + + // set up the comparator so that the source/destination can be compared + indexer := newObjectIndexer() + destinationCleaner, err := newSyncBlobDeleteProcessor(cca) + if err != nil { + return nil, fmt.Errorf("unable to instantiate destination cleaner due to: %s", err.Error()) + } + comparator := newSyncDestinationFilter(indexer, destinationCleaner.removeImmediately) + finalize := func() error { + err = indexer.traverse(transferScheduler.scheduleCopyTransfer, filters) + if err != nil { + return err + } + + anyBlobDeleted := destinationCleaner.wasAnyFileDeleted() jobInitiated, err := transferScheduler.dispatchFinalPart() if err != nil { return err } - if !jobInitiated && !deleteScheduler.wasAnyFileDeleted() { + if !jobInitiated && !anyBlobDeleted { return errors.New("the source and destination are already in sync") + } else if !jobInitiated && anyBlobDeleted { + // some files were deleted but no transfer scheduled + glcm.Exit("the source and destination are now in sync", common.EExitCode.Success()) } cca.setScanningComplete() return nil } - includeFilters := buildIncludeFilters(cca.include) - excludeFilters := buildExcludeFilters(cca.exclude) - - // trigger the progress reporting - cca.waitUntilJobCompletion(false) - - return &syncEnumerator{ - sourceTraverser: sourceTraverser, - destinationTraverser: destinationTraverser, - copyTransferScheduler: transferScheduler, - deleteTransferScheduler: deleteScheduler, - finalize: finalize, - filters: append(includeFilters, excludeFilters...), - }, nil + return newSyncEnumerator(sourceTraverser, destinationTraverser, indexer, filters, comparator, + transferScheduler.scheduleCopyTransfer, finalize), nil } diff --git a/cmd/syncFilter.go b/cmd/syncFilter.go index aa0c5d741..4df8533fa 100644 --- a/cmd/syncFilter.go +++ b/cmd/syncFilter.go @@ -1,71 +1,73 @@ package cmd -import "path" +// with the help of an objectIndexer containing the source objects +// filter out the destination objects that should be transferred +type syncDestinationFilter struct { + // the rejected objects would be passed to the recyclers + recyclers objectProcessor -type excludeFilter struct { - pattern string + // storing the source objects + i *objectIndexer } -func (f *excludeFilter) pass(entity genericEntity) bool { - matched, err := path.Match(f.pattern, entity.name) +func newSyncDestinationFilter(i *objectIndexer, recyclers objectProcessor) objectFilter { + return &syncDestinationFilter{i: i, recyclers: recyclers} +} - // if the pattern failed to match with an error, then we assume the pattern is invalid - // and let it pass - if err != nil { - return true - } +// it will only pass destination objects that are present in the indexer but stale compared to the entry in the map +// if the destinationObject is not present at all, it will be passed to the recyclers +// ex: we already know what the source contains, now we are looking at objects at the destination +// if file x from the destination exists at the source, then we'd only transfer it if it is considered stale compared to its counterpart at the source +// if file x does not exist at the source, then it is considered extra, and will be deleted +func (f *syncDestinationFilter) doesPass(destinationObject storedObject) bool { + storedObjectInMap, present := f.i.indexMap[destinationObject.relativePath] + + // if the destinationObject is present and stale, we let it pass + if present { + defer delete(f.i.indexMap, destinationObject.relativePath) + + if storedObjectInMap.isMoreRecentThan(destinationObject) { + return true + } - if matched { return false } - return true + // purposefully ignore the error from recyclers + // it's a tolerable error, since it just means some extra destination object might hang around a bit longer + _ = f.recyclers(destinationObject) + return false } -func buildExcludeFilters(patterns []string) []entityFilter { - filters := make([]entityFilter, 0) - for _, pattern := range patterns { - filters = append(filters, &excludeFilter{pattern: pattern}) - } +// with the help of an objectIndexer containing the destination objects +// filter out the source objects that should be transferred +type syncSourceFilter struct { - return filters + // storing the destination objects + i *objectIndexer } -// design explanation: -// include filters are different from the exclude ones, which work together in the "AND" manner -// meaning and if an entity is rejected by any of the exclude filters, then it is rejected by all of them -// as a result, the exclude filters can be in their own struct, and work correctly -// on the other hand, include filters work in the "OR" manner -// meaning that if an entity is accepted by any of the include filters, then it is accepted by all of them -// consequently, all the include patterns must be stored together -type includeFilter struct { - patterns []string +func newSyncSourceFilter(i *objectIndexer) objectFilter { + return &syncSourceFilter{i: i} } -func (f *includeFilter) pass(entity genericEntity) bool { - if len(f.patterns) == 0 { - return true - } - - for _, pattern := range f.patterns { - matched, err := path.Match(pattern, entity.name) +// it will only pass items that are: +// 1. not present in the map +// 2. present but is more recent than the entry in the map +// note: we remove the storedObject if it is present +func (f *syncSourceFilter) doesPass(sourceObject storedObject) bool { + storedObjectInMap, present := f.i.indexMap[sourceObject.relativePath] - // if the pattern failed to match with an error, then we assume the pattern is invalid - // and ignore it - if err != nil { - continue - } + // if the sourceObject is more recent, we let it pass + if present { + defer delete(f.i.indexMap, sourceObject.relativePath) - // if an entity is accepted by any of the include filters - // it is accepted - if matched { + if sourceObject.isMoreRecentThan(storedObjectInMap) { return true } - } - return false -} + return false + } -func buildIncludeFilters(patterns []string) []entityFilter { - return []entityFilter{&includeFilter{patterns: patterns}} + return true } diff --git a/cmd/syncIndexer.go b/cmd/syncIndexer.go new file mode 100644 index 000000000..87ccd23d1 --- /dev/null +++ b/cmd/syncIndexer.go @@ -0,0 +1,33 @@ +package cmd + +// the objectIndexer is essential for the generic sync enumerator to work +// it can serve as a: +// 1. objectProcessor: accumulate a lookup map with given storedObjects +// 2. resourceTraverser: go through the entities in the map like a traverser +type objectIndexer struct { + indexMap map[string]storedObject +} + +func newObjectIndexer() *objectIndexer { + indexer := objectIndexer{} + indexer.indexMap = make(map[string]storedObject) + + return &indexer +} + +// process the given stored object by indexing it using its relative path +func (i *objectIndexer) store(storedObject storedObject) (err error) { + i.indexMap[storedObject.relativePath] = storedObject + return +} + +// go through the remaining stored objects in the map to process them +func (i *objectIndexer) traverse(processor objectProcessor, filters []objectFilter) (err error) { + for _, value := range i.indexMap { + err = processIfPassedFilters(filters, value, processor) + if err != nil { + return + } + } + return +} diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index de6a7efe3..3317a6447 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -1,26 +1,21 @@ package cmd import ( + "context" "fmt" + "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-azcopy/ste" + "github.com/Azure/azure-storage-blob-go/azblob" + "net/url" "os" + "path" "path/filepath" - "strings" ) -type syncTransferProcessor struct { - numOfTransfersPerPart int - copyJobTemplate *common.CopyJobPartOrderRequest - source string - destination string - - // keep a handle to initiate progress tracking - cca *cookedSyncCmdArgs -} - -func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) *syncTransferProcessor { - processor := syncTransferProcessor{} - processor.copyJobTemplate = &common.CopyJobPartOrderRequest{ +// extract the right info from cooked arguments and instantiate a generic copy transfer processor from it +func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) *copyTransferProcessor { + copyJobTemplate := &common.CopyJobPartOrderRequest{ JobID: cca.jobID, CommandString: cca.commandString, FromTo: cca.fromTo, @@ -32,150 +27,135 @@ func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) // flags BlobAttributes: common.BlobTransferAttributes{ - PreserveLastModifiedTime: true, + PreserveLastModifiedTime: true, // must be true for sync so that future syncs have this information available MD5ValidationOption: cca.md5ValidationOption, - }, - ForceWrite: true, + BlockSizeInBytes: cca.blockSize}, + ForceWrite: true, // once we decide to transfer for a sync operation, we overwrite the destination regardless LogLevel: cca.logVerbosity, } - // useful for building transfers later - processor.source = cca.source - processor.destination = cca.destination + reportFirstPart := func() { cca.setFirstPartOrdered() } + reportFinalPart := func() { cca.isEnumerationComplete = true } - processor.cca = cca - processor.numOfTransfersPerPart = numOfTransfersPerPart - return &processor + // note that the source and destination, along with the template are given to the generic processor's constructor + // this means that given an object with a relative path, this processor already knows how to schedule the right kind of transfers + return newCopyTransferProcessor(copyJobTemplate, numOfTransfersPerPart, cca.source, cca.destination, reportFirstPart, reportFinalPart) } -func (s *syncTransferProcessor) process(entity genericEntity) (err error) { - if len(s.copyJobTemplate.Transfers) == s.numOfTransfersPerPart { - err = s.sendPartToSte() - if err != nil { - return err - } - - // reset the transfers buffer - s.copyJobTemplate.Transfers = []common.CopyTransfer{} - s.copyJobTemplate.PartNum++ - } +// base for delete processors targeting different resources +type interactiveDeleteProcessor struct { + // the plugged-in deleter that performs the actual deletion + deleter objectProcessor - // only append the transfer after we've checked and dispatched a part - // so that there is at least one transfer for the final part - s.copyJobTemplate.Transfers = append(s.copyJobTemplate.Transfers, common.CopyTransfer{ - Source: s.appendEntityPathToResourcePath(entity.relativePath, s.source), - Destination: s.appendEntityPathToResourcePath(entity.relativePath, s.destination), - SourceSize: entity.size, - LastModifiedTime: entity.lastModifiedTime, - }) - return nil -} - -func (s *syncTransferProcessor) appendEntityPathToResourcePath(entityPath, parentPath string) string { - if entityPath == "" { - return parentPath - } - - return strings.Join([]string{parentPath, entityPath}, common.AZCOPY_PATH_SEPARATOR_STRING) -} - -func (s *syncTransferProcessor) dispatchFinalPart() (copyJobInitiated bool, err error) { - numberOfCopyTransfers := len(s.copyJobTemplate.Transfers) - - // if the number of transfer to copy is - // and no part was dispatched, then it means there is no work to do - if s.copyJobTemplate.PartNum == 0 && numberOfCopyTransfers == 0 { - return false, nil - } - - if numberOfCopyTransfers > 0 { - s.copyJobTemplate.IsFinalPart = true - err = s.sendPartToSte() - if err != nil { - return false, err - } - } - - s.cca.isEnumerationComplete = true - return true, nil -} - -func (s *syncTransferProcessor) sendPartToSte() error { - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(s.copyJobTemplate), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed to dispatch because %s", - s.copyJobTemplate.JobID, s.copyJobTemplate.PartNum, resp.ErrorMsg) - } - - // if the current part order sent to ste is 0, then alert the progress reporting routine - if s.copyJobTemplate.PartNum == 0 { - s.cca.setFirstPartOrdered() - } - - return nil -} - -type syncLocalDeleteProcessor struct { - rootPath string + // whether force delete is on + force bool // ask the user for permission the first time we delete a file hasPromptedUser bool + // used for prompt message + // examples: "blobs", "local files", etc. + objectType string + // note down whether any delete should happen shouldDelete bool - - // keep a handle for progress tracking - cca *cookedSyncCmdArgs } -func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs, isSource bool) *syncLocalDeleteProcessor { - rootPath := cca.source - if !isSource { - rootPath = cca.destination +func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err error) { + if !d.hasPromptedUser { + d.shouldDelete = d.promptForConfirmation() + d.hasPromptedUser = true } - return &syncLocalDeleteProcessor{rootPath: rootPath, cca: cca, hasPromptedUser: false} -} - -func (s *syncLocalDeleteProcessor) process(entity genericEntity) (err error) { - if !s.hasPromptedUser { - s.shouldDelete = s.promptForConfirmation() - } - - if !s.shouldDelete { + if !d.shouldDelete { return nil } - err = os.Remove(filepath.Join(s.rootPath, entity.relativePath)) + err = d.deleter(object) if err != nil { - glcm.Info(fmt.Sprintf("error %s deleting the file %s", err.Error(), entity.relativePath)) + glcm.Info(fmt.Sprintf("error %s deleting the object %s", err.Error(), object.relativePath)) } return } -func (s *syncLocalDeleteProcessor) promptForConfirmation() (shouldDelete bool) { +func (d *interactiveDeleteProcessor) promptForConfirmation() (shouldDelete bool) { shouldDelete = false // omit asking if the user has already specified - if s.cca.force { + if d.force { shouldDelete = true } else { - answer := glcm.Prompt(fmt.Sprintf("Sync has discovered local files that are not present at the source, would you like to delete them? Please confirm with y/n: ")) + answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them? Please confirm with y/n: ", d.objectType)) if answer == "y" || answer == "yes" { shouldDelete = true - glcm.Info("Confirmed. The extra local files will be deleted.") + glcm.Info(fmt.Sprintf("Confirmed. The extra %s will be deleted:", d.objectType)) } else { glcm.Info("No deletions will happen.") } } - - s.hasPromptedUser = true return } -func (s *syncLocalDeleteProcessor) wasAnyFileDeleted() bool { - // we'd have prompted the user if any entity was passed in - return s.hasPromptedUser +func (d *interactiveDeleteProcessor) wasAnyFileDeleted() bool { + // we'd have prompted the user if any stored object was passed in + return d.hasPromptedUser +} + +func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs) *interactiveDeleteProcessor { + localDeleter := localFileDeleter{rootPath: cca.destination} + return &interactiveDeleteProcessor{deleter: localDeleter.deleteFile, force: cca.force, objectType: "local files"} +} + +type localFileDeleter struct { + rootPath string +} + +func (l *localFileDeleter) deleteFile(object storedObject) error { + glcm.Info("Deleting file: " + object.relativePath) + return os.Remove(filepath.Join(l.rootPath, object.relativePath)) +} + +func newSyncBlobDeleteProcessor(cca *cookedSyncCmdArgs) (*interactiveDeleteProcessor, error) { + rawURL, err := url.Parse(cca.destination) + if err != nil { + return nil, err + } else if err == nil && cca.destinationSAS != "" { + copyHandlerUtil{}.appendQueryParamToUrl(rawURL, cca.destinationSAS) + } + + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p, err := createBlobPipeline(ctx, cca.credentialInfo) + if err != nil { + return nil, err + } + + return &interactiveDeleteProcessor{deleter: newBlobDeleter(rawURL, p, ctx).deleteBlob, force: cca.force, objectType: "blobs"}, nil +} + +type blobDeleter struct { + rootURL *url.URL + p pipeline.Pipeline + ctx context.Context +} + +func newBlobDeleter(rawRootURL *url.URL, p pipeline.Pipeline, ctx context.Context) *blobDeleter { + return &blobDeleter{ + rootURL: rawRootURL, + p: p, + ctx: ctx, + } +} + +func (b *blobDeleter) deleteBlob(object storedObject) error { + glcm.Info("Deleting: " + object.relativePath) + + // construct the blob URL using its relative path + // the rootURL could be pointing to a container, or a virtual directory + blobURLParts := azblob.NewBlobURLParts(*b.rootURL) + blobURLParts.BlobName = path.Join(blobURLParts.BlobName, object.relativePath) + + blobURL := azblob.NewBlobURL(blobURLParts.URL(), b.p) + _, err := blobURL.Delete(b.ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) + return err } diff --git a/cmd/syncTraverser.go b/cmd/syncTraverser.go index 6dd2ef6ea..0b9e9def9 100644 --- a/cmd/syncTraverser.go +++ b/cmd/syncTraverser.go @@ -2,134 +2,57 @@ package cmd import ( "context" - "fmt" - "github.com/Azure/azure-pipeline-go/pipeline" - "github.com/Azure/azure-storage-azcopy/common" + "errors" "github.com/Azure/azure-storage-azcopy/ste" - "github.com/Azure/azure-storage-blob-go/azblob" - "io/ioutil" "net/url" - "os" - "path/filepath" "strings" "sync/atomic" ) -// -------------------------------------- Traversers -------------------------------------- \\ -// these traversers allow us to iterate through different resource types +func newLocalTraverserForSync(cca *cookedSyncCmdArgs, isSource bool) (*localTraverser, error) { + var fullPath string -type blobTraverser struct { - rawURL *url.URL - p pipeline.Pipeline - ctx context.Context - recursive bool - isSource bool - cca *cookedSyncCmdArgs -} - -func (blobTraverser *blobTraverser) getPropertiesIfSingleBlob() (blobProps *azblob.BlobGetPropertiesResponse, isBlob bool) { - blobURL := azblob.NewBlobURL(*blobTraverser.rawURL, blobTraverser.p) - blobProps, blobPropertiesErr := blobURL.GetProperties(blobTraverser.ctx, azblob.BlobAccessConditions{}) - - // if there was no problem getting the properties, it means that we are looking at a single blob - if blobPropertiesErr == nil { - isBlob = true - return - } - - return -} - -func (blobTraverser *blobTraverser) traverse(processor entityProcessor, filters []entityFilter) (err error) { - blobUrlParts := azblob.NewBlobURLParts(*blobTraverser.rawURL) - - // check if the url points to a single blob - blobProperties, isBlob := blobTraverser.getPropertiesIfSingleBlob() - if isBlob { - entity := genericEntity{ - name: getEntityNameOnly(blobUrlParts.BlobName), - relativePath: "", // relative path makes no sense when the full path already points to the file - lastModifiedTime: blobProperties.LastModified(), - size: blobProperties.ContentLength(), - } - blobTraverser.incrementEnumerationCounter() - return processIfPassedFilters(filters, entity, processor) + if isSource { + fullPath = cca.source + } else { + fullPath = cca.destination } - // get the container URL so that we can list the blobs - containerRawURL := copyHandlerUtil{}.getContainerUrl(blobUrlParts) - containerURL := azblob.NewContainerURL(containerRawURL, blobTraverser.p) - - // get the search prefix to aid in the listing - searchPrefix := blobUrlParts.BlobName - - // append a slash if it is not already present - if searchPrefix != "" && !strings.HasSuffix(searchPrefix, common.AZCOPY_PATH_SEPARATOR_STRING) { - searchPrefix += common.AZCOPY_PATH_SEPARATOR_STRING + if strings.ContainsAny(fullPath, "*?") { + return nil, errors.New("illegal local path, no pattern matching allowed for sync command") } - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix - listBlob, err := containerURL.ListBlobsFlatSegment(blobTraverser.ctx, marker, - azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs. Failed with error %s", err.Error()) - } - - // process the blobs returned in this result segment - for _, blobInfo := range listBlob.Segment.BlobItems { - relativePath := strings.Replace(blobInfo.Name, searchPrefix, "", 1) - - // if recursive - if !blobTraverser.recursive && strings.Contains(relativePath, common.AZCOPY_PATH_SEPARATOR_STRING) { - continue - } + incrementEnumerationCounter := func() { + var counterAddr *uint64 - entity := genericEntity{ - name: getEntityNameOnly(blobInfo.Name), - relativePath: relativePath, - lastModifiedTime: blobInfo.Properties.LastModified, - size: *blobInfo.Properties.ContentLength, - } - blobTraverser.incrementEnumerationCounter() - processErr := processIfPassedFilters(filters, entity, processor) - if processErr != nil { - return processErr - } + if isSource { + counterAddr = &cca.atomicSourceFilesScanned + } else { + counterAddr = &cca.atomicDestinationFilesScanned } - marker = listBlob.NextMarker + atomic.AddUint64(counterAddr, 1) } - return -} - -func (blobTraverser *blobTraverser) incrementEnumerationCounter() { - var counterAddr *uint64 - - if blobTraverser.isSource { - counterAddr = &blobTraverser.cca.atomicSourceFilesScanned - } else { - counterAddr = &blobTraverser.cca.atomicDestinationFilesScanned - } + traverser := newLocalTraverser(fullPath, cca.recursive, incrementEnumerationCounter) - atomic.AddUint64(counterAddr, 1) + return traverser, nil } -func newBlobTraverser(cca *cookedSyncCmdArgs, isSource bool) (traverser *blobTraverser, err error) { - traverser = &blobTraverser{} +func newBlobTraverserForSync(cca *cookedSyncCmdArgs, isSource bool) (t *blobTraverser, err error) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + // figure out the right URL + var rawURL *url.URL if isSource { - traverser.rawURL, err = url.Parse(cca.source) + rawURL, err = url.Parse(cca.source) if err == nil && cca.sourceSAS != "" { - copyHandlerUtil{}.appendQueryParamToUrl(traverser.rawURL, cca.sourceSAS) + copyHandlerUtil{}.appendQueryParamToUrl(rawURL, cca.sourceSAS) } - } else { - traverser.rawURL, err = url.Parse(cca.destination) + rawURL, err = url.Parse(cca.destination) if err == nil && cca.destinationSAS != "" { - copyHandlerUtil{}.appendQueryParamToUrl(traverser.rawURL, cca.destinationSAS) + copyHandlerUtil{}.appendQueryParamToUrl(rawURL, cca.destinationSAS) } } @@ -137,132 +60,26 @@ func newBlobTraverser(cca *cookedSyncCmdArgs, isSource bool) (traverser *blobTra return } - traverser.p, err = createBlobPipeline(ctx, cca.credentialInfo) - if err != nil { - return + if strings.Contains(rawURL.Path, "*") { + return nil, errors.New("illegal URL, no pattern matching allowed for sync command") } - traverser.isSource = isSource - traverser.ctx = context.TODO() - traverser.recursive = cca.recursive - traverser.cca = cca - return -} - -type localTraverser struct { - fullPath string - recursive bool - followSymlinks bool - isSource bool - cca *cookedSyncCmdArgs -} - -func (localTraverser *localTraverser) traverse(processor entityProcessor, filters []entityFilter) (err error) { - singleFileInfo, isSingleFile, err := localTraverser.getInfoIfSingleFile() - + p, err := createBlobPipeline(ctx, cca.credentialInfo) if err != nil { - return fmt.Errorf("cannot scan the path %s, please verify that it is a valid", localTraverser.fullPath) - } - - // if the path is a single file, then pass it through the filters and send to processor - if isSingleFile { - localTraverser.incrementEnumerationCounter() - err = processIfPassedFilters(filters, genericEntity{ - name: singleFileInfo.Name(), - relativePath: "", // relative path makes no sense when the full path already points to the file - lastModifiedTime: singleFileInfo.ModTime(), - size: singleFileInfo.Size()}, processor) return + } - } else { - if localTraverser.recursive { - err = filepath.Walk(localTraverser.fullPath, func(filePath string, fileInfo os.FileInfo, fileError error) error { - if fileError != nil { - return fileError - } - - // skip the subdirectories - if fileInfo.IsDir() { - return nil - } - - localTraverser.incrementEnumerationCounter() - return processIfPassedFilters(filters, genericEntity{ - name: fileInfo.Name(), - relativePath: strings.Replace(filePath, localTraverser.fullPath+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1), - lastModifiedTime: fileInfo.ModTime(), - size: fileInfo.Size()}, processor) - }) + incrementEnumerationCounter := func() { + var counterAddr *uint64 - return + if isSource { + counterAddr = &cca.atomicSourceFilesScanned } else { - // if recursive is off, we only need to scan the files immediately under the fullPath - files, err := ioutil.ReadDir(localTraverser.fullPath) - if err != nil { - return err - } - - // go through the files and return if any of them fail to process - for _, singleFile := range files { - if singleFile.IsDir() { - continue - } - - localTraverser.incrementEnumerationCounter() - err = processIfPassedFilters(filters, genericEntity{ - name: singleFile.Name(), - relativePath: singleFile.Name(), - lastModifiedTime: singleFile.ModTime(), - size: singleFile.Size()}, processor) - - if err != nil { - return err - } - } + counterAddr = &cca.atomicDestinationFilesScanned } - } - - return -} - -func (localTraverser *localTraverser) getInfoIfSingleFile() (os.FileInfo, bool, error) { - fileInfo, err := os.Stat(localTraverser.fullPath) - - if err != nil { - return nil, false, err - } - - if fileInfo.IsDir() { - return nil, false, nil - } - return fileInfo, true, nil -} - -func (localTraverser *localTraverser) incrementEnumerationCounter() { - var counterAddr *uint64 - - if localTraverser.isSource { - counterAddr = &localTraverser.cca.atomicSourceFilesScanned - } else { - counterAddr = &localTraverser.cca.atomicDestinationFilesScanned - } - - atomic.AddUint64(counterAddr, 1) -} - -func newLocalTraverser(cca *cookedSyncCmdArgs, isSource bool) *localTraverser { - traverser := localTraverser{} - - if isSource { - traverser.fullPath = cca.source - } else { - traverser.fullPath = cca.destination + atomic.AddUint64(counterAddr, 1) } - traverser.isSource = isSource - traverser.recursive = cca.recursive - traverser.followSymlinks = cca.followSymlinks - traverser.cca = cca - return &traverser + return newBlobTraverser(rawURL, p, ctx, cca.recursive, incrementEnumerationCounter), nil } diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go new file mode 100644 index 000000000..9bf6898c2 --- /dev/null +++ b/cmd/zc_enumerator.go @@ -0,0 +1,159 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + "strings" + "time" +) + +// -------------------------------------- Component Definitions -------------------------------------- \\ +// the following interfaces and structs allow the sync enumerator +// to be generic and has as little duplicated code as possible + +// represent a local or remote resource object (ex: local file, blob, etc.) +// we can add more properties if needed, as this is easily extensible +type storedObject struct { + name string + lastModifiedTime time.Time + size int64 + md5 []byte + + // partial path relative to its root directory + // example: rootDir=/var/a/b/c fullPath=/var/a/b/c/d/e/f.pdf => relativePath=d/e/f.pdf name=f.pdf + relativePath string +} + +func (storedObject *storedObject) isMoreRecentThan(storedObject2 storedObject) bool { + return storedObject.lastModifiedTime.After(storedObject2.lastModifiedTime) +} + +// a constructor is used so that in case the storedObject has to change, the callers would get a compilation error +func newStoredObject(name string, relativePath string, lmt time.Time, size int64, md5 []byte) storedObject { + return storedObject{ + name: name, + relativePath: relativePath, + lastModifiedTime: lmt, + size: size, + md5: md5, + } +} + +// capable of traversing a structured resource like container or local directory +// pass each storedObject to the given objectProcessor if it passes all the filters +type resourceTraverser interface { + traverse(processor objectProcessor, filters []objectFilter) error +} + +// given a storedObject, process it accordingly +type objectProcessor func(storedObject storedObject) error + +// given a storedObject, verify if it satisfies the defined conditions +// if yes, return true +type objectFilter interface { + doesPass(storedObject storedObject) bool +} + +// -------------------------------------- Generic Enumerators -------------------------------------- \\ +// the following enumerators must be instantiated with configurations +// they define the work flow in the most generic terms + +type syncEnumerator struct { + // these allow us to go through the source and destination + // there is flexibility in which side we scan first, it could be either the source or the destination + primaryTraverser resourceTraverser + secondaryTraverser resourceTraverser + + // the results from the primary traverser would be stored here + objectIndexer *objectIndexer + + // general filters apply to both the primary and secondary traverser + filters []objectFilter + + // a special filters that apply only to the secondary traverser + // it filters objects as scanning happens, based on the data from the primary traverser stored in the objectIndexer + objectComparator objectFilter + + // the processor responsible for scheduling copy transfers + copyTransferScheduler objectProcessor + + // a finalizer that is always called if the enumeration finishes properly + finalize func() error +} + +func newSyncEnumerator(primaryTraverser, secondaryTraverser resourceTraverser, indexer *objectIndexer, + filters []objectFilter, comparator objectFilter, copyTransferScheduler objectProcessor, finalize func() error) *syncEnumerator { + return &syncEnumerator{ + primaryTraverser: primaryTraverser, + secondaryTraverser: secondaryTraverser, + objectIndexer: indexer, + filters: filters, + objectComparator: comparator, + copyTransferScheduler: copyTransferScheduler, + finalize: finalize, + } +} + +func (e *syncEnumerator) enumerate() (err error) { + // enumerate the primary resource and build lookup map + err = e.primaryTraverser.traverse(e.objectIndexer.store, e.filters) + if err != nil { + return + } + + // add the objectComparator as an extra filter to the list + // so that it can filter given objects based on what's already indexed + e.filters = append(e.filters, e.objectComparator) + + // enumerate the secondary resource and as the objects pass the filters + // they will be scheduled so that transferring can start while scanning is ongoing + err = e.secondaryTraverser.traverse(e.copyTransferScheduler, e.filters) + if err != nil { + return + } + + // execute the finalize func which may perform useful clean up steps + err = e.finalize() + if err != nil { + return + } + + return +} + +// -------------------------------------- Helper Funcs -------------------------------------- \\ + +func passedFilters(filters []objectFilter, storedObject storedObject) bool { + if filters != nil && len(filters) > 0 { + // loop through the filters, if any of them fail, then return false + for _, filter := range filters { + if !filter.doesPass(storedObject) { + return false + } + } + } + + return true +} + +func processIfPassedFilters(filters []objectFilter, storedObject storedObject, processor objectProcessor) (err error) { + if passedFilters(filters, storedObject) { + err = processor(storedObject) + } + + return +} + +// storedObject names are useful for filters +func getObjectNameOnly(fullPath string) (nameOnly string) { + lastPathSeparator := strings.LastIndex(fullPath, common.AZCOPY_PATH_SEPARATOR_STRING) + + // if there is a path separator and it is not the last character + if lastPathSeparator > 0 && lastPathSeparator != len(fullPath)-1 { + // then we separate out the name of the storedObject + nameOnly = fullPath[lastPathSeparator+1:] + } else { + nameOnly = fullPath + } + + return +} diff --git a/cmd/zc_filter.go b/cmd/zc_filter.go new file mode 100644 index 000000000..76b25e888 --- /dev/null +++ b/cmd/zc_filter.go @@ -0,0 +1,80 @@ +package cmd + +import "path" + +type excludeFilter struct { + pattern string +} + +func (f *excludeFilter) doesPass(storedObject storedObject) bool { + matched, err := path.Match(f.pattern, storedObject.name) + + // if the pattern failed to match with an error, then we assume the pattern is invalid + // and let it pass + if err != nil { + return true + } + + if matched { + return false + } + + return true +} + +func buildExcludeFilters(patterns []string) []objectFilter { + filters := make([]objectFilter, 0) + for _, pattern := range patterns { + if pattern != "" { + filters = append(filters, &excludeFilter{pattern: pattern}) + } + } + + return filters +} + +// design explanation: +// include filters are different from the exclude ones, which work together in the "AND" manner +// meaning and if an storedObject is rejected by any of the exclude filters, then it is rejected by all of them +// as a result, the exclude filters can be in their own struct, and work correctly +// on the other hand, include filters work in the "OR" manner +// meaning that if an storedObject is accepted by any of the include filters, then it is accepted by all of them +// consequently, all the include patterns must be stored together +type includeFilter struct { + patterns []string +} + +func (f *includeFilter) doesPass(storedObject storedObject) bool { + if len(f.patterns) == 0 { + return true + } + + for _, pattern := range f.patterns { + matched, err := path.Match(pattern, storedObject.name) + + // if the pattern failed to match with an error, then we assume the pattern is invalid + // and ignore it + if err != nil { + continue + } + + // if an storedObject is accepted by any of the include filters + // it is accepted + if matched { + return true + } + } + + return false +} + +func buildIncludeFilters(patterns []string) []objectFilter { + validPatterns := make([]string, 0) + for _, pattern := range patterns { + if pattern != "" { + validPatterns = append(validPatterns, pattern) + } + } + + return []objectFilter{&includeFilter{patterns: validPatterns}} +} diff --git a/cmd/zc_processor.go b/cmd/zc_processor.go new file mode 100644 index 000000000..66b2d54b4 --- /dev/null +++ b/cmd/zc_processor.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "fmt" + "github.com/Azure/azure-storage-azcopy/common" + "strings" +) + +type copyTransferProcessor struct { + numOfTransfersPerPart int + copyJobTemplate *common.CopyJobPartOrderRequest + source string + destination string + + // handles for progress tracking + reportFirstPartDispatched func() + reportFinalPartDispatched func() +} + +func newCopyTransferProcessor(copyJobTemplate *common.CopyJobPartOrderRequest, numOfTransfersPerPart int, + source string, destination string, reportFirstPartDispatched func(), reportFinalPartDispatched func()) *copyTransferProcessor { + return ©TransferProcessor{ + numOfTransfersPerPart: numOfTransfersPerPart, + copyJobTemplate: copyJobTemplate, + source: source, + destination: destination, + reportFirstPartDispatched: reportFirstPartDispatched, + reportFinalPartDispatched: reportFinalPartDispatched, + } +} + +func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject storedObject) (err error) { + if len(s.copyJobTemplate.Transfers) == s.numOfTransfersPerPart { + err = s.sendPartToSte() + if err != nil { + return err + } + + // reset the transfers buffer + s.copyJobTemplate.Transfers = []common.CopyTransfer{} + s.copyJobTemplate.PartNum++ + } + + // only append the transfer after we've checked and dispatched a part + // so that there is at least one transfer for the final part + s.copyJobTemplate.Transfers = append(s.copyJobTemplate.Transfers, common.CopyTransfer{ + Source: s.appendObjectPathToResourcePath(storedObject.relativePath, s.source), + Destination: s.appendObjectPathToResourcePath(storedObject.relativePath, s.destination), + SourceSize: storedObject.size, + LastModifiedTime: storedObject.lastModifiedTime, + ContentMD5: storedObject.md5, + }) + return nil +} + +func (s *copyTransferProcessor) appendObjectPathToResourcePath(storedObjectPath, parentPath string) string { + if storedObjectPath == "" { + return parentPath + } + + return strings.Join([]string{parentPath, storedObjectPath}, common.AZCOPY_PATH_SEPARATOR_STRING) +} + +func (s *copyTransferProcessor) dispatchFinalPart() (copyJobInitiated bool, err error) { + numberOfCopyTransfers := len(s.copyJobTemplate.Transfers) + + // if the number of transfer to copy is 0 + // and no part was dispatched, then it means there is no work to do + if s.copyJobTemplate.PartNum == 0 && numberOfCopyTransfers == 0 { + return false, nil + } + + if numberOfCopyTransfers > 0 { + s.copyJobTemplate.IsFinalPart = true + err = s.sendPartToSte() + if err != nil { + return false, err + } + } + + if s.reportFinalPartDispatched != nil { + s.reportFinalPartDispatched() + } + return true, nil +} + +func (s *copyTransferProcessor) sendPartToSte() error { + var resp common.CopyJobPartOrderResponse + Rpc(common.ERpcCmd.CopyJobPartOrder(), s.copyJobTemplate, &resp) + if !resp.JobStarted { + return fmt.Errorf("copy job part order with JobId %s and part number %d failed to dispatch because %s", + s.copyJobTemplate.JobID, s.copyJobTemplate.PartNum, resp.ErrorMsg) + } + + // if the current part order sent to ste is 0, then alert the progress reporting routine + if s.copyJobTemplate.PartNum == 0 && s.reportFirstPartDispatched != nil { + s.reportFirstPartDispatched() + } + + return nil +} diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go new file mode 100644 index 000000000..a1bebe732 --- /dev/null +++ b/cmd/zc_traverser_blob.go @@ -0,0 +1,115 @@ +package cmd + +import ( + "context" + "fmt" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + "net/url" + "strings" +) + +// allow us to iterate through a path pointing to the blob endpoint +type blobTraverser struct { + rawURL *url.URL + p pipeline.Pipeline + ctx context.Context + recursive bool + + // a generic function to notify that a new stored object has been enumerated + incrementEnumerationCounter func() +} + +func (t *blobTraverser) getPropertiesIfSingleBlob() (blobProps *azblob.BlobGetPropertiesResponse, isBlob bool) { + blobURL := azblob.NewBlobURL(*t.rawURL, t.p) + blobProps, blobPropertiesErr := blobURL.GetProperties(t.ctx, azblob.BlobAccessConditions{}) + + // if there was no problem getting the properties, it means that we are looking at a single blob + if blobPropertiesErr == nil { + isBlob = true + return + } + + return +} + +func (t *blobTraverser) traverse(processor objectProcessor, filters []objectFilter) (err error) { + blobUrlParts := azblob.NewBlobURLParts(*t.rawURL) + util := copyHandlerUtil{} + + // check if the url points to a single blob + blobProperties, isBlob := t.getPropertiesIfSingleBlob() + if isBlob { + storedObject := newStoredObject( + getObjectNameOnly(blobUrlParts.BlobName), + "", // relative path makes no sense when the full path already points to the file + blobProperties.LastModified(), + blobProperties.ContentLength(), + blobProperties.ContentMD5(), + ) + t.incrementEnumerationCounter() + return processIfPassedFilters(filters, storedObject, processor) + } + + // get the container URL so that we can list the blobs + containerRawURL := copyHandlerUtil{}.getContainerUrl(blobUrlParts) + containerURL := azblob.NewContainerURL(containerRawURL, t.p) + + // get the search prefix to aid in the listing + // example: for a url like https://test.blob.core.windows.net/test/foo/bar/bla + // the search prefix would be foo/bar/bla + searchPrefix := blobUrlParts.BlobName + + // append a slash if it is not already present + // example: foo/bar/bla becomes foo/bar/bla/ so that we only list children of the virtual directory + if searchPrefix != "" && !strings.HasSuffix(searchPrefix, common.AZCOPY_PATH_SEPARATOR_STRING) { + searchPrefix += common.AZCOPY_PATH_SEPARATOR_STRING + } + + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix + // TODO optimize for the case where recursive is off + listBlob, err := containerURL.ListBlobsFlatSegment(t.ctx, marker, + azblob.ListBlobsSegmentOptions{Prefix: searchPrefix, Details: azblob.BlobListingDetails{Metadata: true}}) + if err != nil { + return fmt.Errorf("cannot list blobs. Failed with error %s", err.Error()) + } + + // process the blobs returned in this result segment + for _, blobInfo := range listBlob.Segment.BlobItems { + // if the blob represents a hdi folder, then skip it + if util.doesBlobRepresentAFolder(blobInfo.Metadata) { + continue + } + + relativePath := strings.Replace(blobInfo.Name, searchPrefix, "", 1) + + // if recursive + if !t.recursive && strings.Contains(relativePath, common.AZCOPY_PATH_SEPARATOR_STRING) { + continue + } + + storedObject := storedObject{ + name: getObjectNameOnly(blobInfo.Name), + relativePath: relativePath, + lastModifiedTime: blobInfo.Properties.LastModified, + size: *blobInfo.Properties.ContentLength, + } + t.incrementEnumerationCounter() + processErr := processIfPassedFilters(filters, storedObject, processor) + if processErr != nil { + return processErr + } + } + + marker = listBlob.NextMarker + } + + return +} + +func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive bool, incrementEnumerationCounter func()) (t *blobTraverser) { + t = &blobTraverser{rawURL: rawURL, p: p, ctx: ctx, recursive: recursive, incrementEnumerationCounter: incrementEnumerationCounter} + return +} diff --git a/cmd/zc_traverser_local.go b/cmd/zc_traverser_local.go new file mode 100644 index 000000000..7745e2338 --- /dev/null +++ b/cmd/zc_traverser_local.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "fmt" + "github.com/Azure/azure-storage-azcopy/common" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +type localTraverser struct { + fullPath string + recursive bool + + // a generic function to notify that a new stored object has been enumerated + incrementEnumerationCounter func() +} + +func (t *localTraverser) traverse(processor objectProcessor, filters []objectFilter) (err error) { + singleFileInfo, isSingleFile, err := t.getInfoIfSingleFile() + + if err != nil { + return fmt.Errorf("cannot scan the path %s, please verify that it is a valid", t.fullPath) + } + + // if the path is a single file, then pass it through the filters and send to processor + if isSingleFile { + t.incrementEnumerationCounter() + err = processIfPassedFilters(filters, newStoredObject(singleFileInfo.Name(), + "", // relative path makes no sense when the full path already points to the file + singleFileInfo.ModTime(), singleFileInfo.Size(), nil), processor) + return + + } else { + if t.recursive { + err = filepath.Walk(t.fullPath, func(filePath string, fileInfo os.FileInfo, fileError error) error { + if fileError != nil { + return fileError + } + + // skip the subdirectories + if fileInfo.IsDir() { + return nil + } + + t.incrementEnumerationCounter() + return processIfPassedFilters(filters, newStoredObject(fileInfo.Name(), + strings.Replace(filePath, t.fullPath+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1), + fileInfo.ModTime(), + fileInfo.Size(), nil), processor) + }) + + return + } else { + // if recursive is off, we only need to scan the files immediately under the fullPath + files, err := ioutil.ReadDir(t.fullPath) + if err != nil { + return err + } + + // go through the files and return if any of them fail to process + for _, singleFile := range files { + if singleFile.IsDir() { + continue + } + + t.incrementEnumerationCounter() + err = processIfPassedFilters(filters, newStoredObject(singleFile.Name(), singleFile.Name(), singleFile.ModTime(), singleFile.Size(), nil), processor) + + if err != nil { + return err + } + } + } + } + + return +} + +func (t *localTraverser) getInfoIfSingleFile() (os.FileInfo, bool, error) { + fileInfo, err := os.Stat(t.fullPath) + + if err != nil { + return nil, false, err + } + + if fileInfo.IsDir() { + return nil, false, nil + } + + return fileInfo, true, nil +} + +func newLocalTraverser(fullPath string, recursive bool, incrementEnumerationCounter func()) *localTraverser { + traverser := localTraverser{ + fullPath: fullPath, + recursive: recursive, + incrementEnumerationCounter: incrementEnumerationCounter} + return &traverser +} diff --git a/cmd/zt_generic_filter_test.go b/cmd/zt_generic_filter_test.go new file mode 100644 index 000000000..86903778f --- /dev/null +++ b/cmd/zt_generic_filter_test.go @@ -0,0 +1,55 @@ +package cmd + +import ( + chk "gopkg.in/check.v1" +) + +type genericFilterSuite struct{} + +var _ = chk.Suite(&genericFilterSuite{}) + +func (s *genericFilterSuite) TestIncludeFilter(c *chk.C) { + // set up the filters + raw := rawSyncCmdArgs{} + includePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") + includeFilter := buildIncludeFilters(includePatternList)[0] + + // test the positive cases + filesToPass := []string{"bla.pdf", "fancy.jpeg", "socool.jpeg.pdf", "exactName"} + for _, file := range filesToPass { + passed := includeFilter.doesPass(storedObject{name: file}) + c.Assert(passed, chk.Equals, true) + } + + // test the negative cases + filesNotToPass := []string{"bla.pdff", "fancyjpeg", "socool.jpeg.pdf.wut", "eexactName"} + for _, file := range filesNotToPass { + passed := includeFilter.doesPass(storedObject{name: file}) + c.Assert(passed, chk.Equals, false) + } +} + +func (s *genericFilterSuite) TestExcludeFilter(c *chk.C) { + // set up the filters + raw := rawSyncCmdArgs{} + excludePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") + excludeFilterList := buildExcludeFilters(excludePatternList) + + // test the positive cases + filesToPass := []string{"bla.pdfe", "fancy.jjpeg", "socool.png", "eexactName"} + for _, file := range filesToPass { + dummyProcessor := &dummyProcessor{} + err := processIfPassedFilters(excludeFilterList, storedObject{name: file}, dummyProcessor.process) + c.Assert(err, chk.IsNil) + c.Assert(len(dummyProcessor.record), chk.Equals, 1) + } + + // test the negative cases + filesToNotPass := []string{"bla.pdf", "fancy.jpeg", "socool.jpeg.pdf", "exactName"} + for _, file := range filesToNotPass { + dummyProcessor := &dummyProcessor{} + err := processIfPassedFilters(excludeFilterList, storedObject{name: file}, dummyProcessor.process) + c.Assert(err, chk.IsNil) + c.Assert(len(dummyProcessor.record), chk.Equals, 0) + } +} diff --git a/cmd/zt_generic_processor_test.go b/cmd/zt_generic_processor_test.go new file mode 100644 index 000000000..b2c3b2cf9 --- /dev/null +++ b/cmd/zt_generic_processor_test.go @@ -0,0 +1,122 @@ +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" + "path/filepath" + "time" +) + +type genericProcessorSuite struct{} + +var _ = chk.Suite(&genericProcessorSuite{}) + +type processorTestSuiteHelper struct{} + +// return a list of sample entities +func (processorTestSuiteHelper) getSampleObjectList() []storedObject { + return []storedObject{ + {name: "file1", relativePath: "file1", lastModifiedTime: time.Now()}, + {name: "file2", relativePath: "file2", lastModifiedTime: time.Now()}, + {name: "file3", relativePath: "sub1/file3", lastModifiedTime: time.Now()}, + {name: "file4", relativePath: "sub1/file4", lastModifiedTime: time.Now()}, + {name: "file5", relativePath: "sub1/sub2/file5", lastModifiedTime: time.Now()}, + {name: "file6", relativePath: "sub1/sub2/file6", lastModifiedTime: time.Now()}, + } +} + +// given a list of entities, return the relative paths in a list, to help with validations +func (processorTestSuiteHelper) getExpectedTransferFromStoredObjectList(storedObjectList []storedObject) []string { + expectedTransfers := make([]string, 0) + for _, storedObject := range storedObjectList { + expectedTransfers = append(expectedTransfers, storedObject.relativePath) + } + + return expectedTransfers +} + +func (processorTestSuiteHelper) getCopyJobTemplate() *common.CopyJobPartOrderRequest { + return &common.CopyJobPartOrderRequest{} +} + +func (s *genericProcessorSuite) TestCopyTransferProcessorMultipleFiles(c *chk.C) { + bsu := getBSU() + + // set up source and destination + containerURL, _ := getContainerURL(c, bsu) + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // exercise the processor + sampleObjects := processorTestSuiteHelper{}.getSampleObjectList() + for _, numOfParts := range []int{1, 3} { + numOfTransfersPerPart := len(sampleObjects) / numOfParts + copyProcessor := newCopyTransferProcessor(processorTestSuiteHelper{}.getCopyJobTemplate(), numOfTransfersPerPart, + containerURL.String(), dstDirName, nil, nil) + + // go through the objects and make sure they are processed without error + for _, storedObject := range sampleObjects { + err := copyProcessor.scheduleCopyTransfer(storedObject) + c.Assert(err, chk.IsNil) + } + + // make sure everything has been dispatched apart from the final one + c.Assert(copyProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(numOfParts-1)) + + // dispatch final part + jobInitiated, err := copyProcessor.dispatchFinalPart() + c.Assert(jobInitiated, chk.Equals, true) + c.Assert(err, chk.IsNil) + + // assert the right transfers were scheduled + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, + processorTestSuiteHelper{}.getExpectedTransferFromStoredObjectList(sampleObjects), mockedRPC) + + mockedRPC.reset() + } +} + +func (s *genericProcessorSuite) TestCopyTransferProcessorSingleFile(c *chk.C) { + bsu := getBSU() + containerURL, _ := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up the container with a single blob + blobList := []string{"singlefile101"} + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + c.Assert(containerURL, chk.NotNil) + + // set up the directory with a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobList[0] + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // set up the processor + copyProcessor := newCopyTransferProcessor(processorTestSuiteHelper{}.getCopyJobTemplate(), 2, + containerURL.NewBlockBlobURL(blobList[0]).String(), filepath.Join(dstDirName, dstFileName), nil, nil) + + // exercise the copy transfer processor + storedObject := newStoredObject(blobList[0], "", time.Now(), 0, nil) + err := copyProcessor.scheduleCopyTransfer(storedObject) + c.Assert(err, chk.IsNil) + + // no part should have been dispatched + c.Assert(copyProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(0)) + + // dispatch final part + jobInitiated, err := copyProcessor.dispatchFinalPart() + c.Assert(jobInitiated, chk.Equals, true) + + // assert the right transfers were scheduled + validateTransfersAreScheduled(c, containerURL.String(), dstDirName, + blobList, mockedRPC) +} diff --git a/cmd/zt_sync_traverser_test.go b/cmd/zt_generic_traverser_test.go similarity index 52% rename from cmd/zt_sync_traverser_test.go rename to cmd/zt_generic_traverser_test.go index 7cd7528df..8b11b37e5 100644 --- a/cmd/zt_sync_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -1,27 +1,22 @@ package cmd import ( + "context" "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-azcopy/ste" + "github.com/Azure/azure-storage-blob-go/azblob" chk "gopkg.in/check.v1" - "path" "path/filepath" "strings" ) -type syncTraverserSuite struct{} +type genericTraverserSuite struct{} -var _ = chk.Suite(&syncTraverserSuite{}) +var _ = chk.Suite(&genericTraverserSuite{}) -type dummyProcessor struct { - record []genericEntity -} - -func (d *dummyProcessor) process(entity genericEntity) (err error) { - d.record = append(d.record, entity) - return -} - -func (s *syncTraverserSuite) TestSyncTraverserSingleEntity(c *chk.C) { +// validate traversing a single blob and a single file +// compare that blob and local traversers get consistent results +func (s *genericTraverserSuite) TestTraverserWithSingleObject(c *chk.C) { bsu := getBSU() containerURL, containerName := createNewContainer(c, bsu) defer deleteContainer(c, containerURL) @@ -38,28 +33,24 @@ func (s *syncTraverserSuite) TestSyncTraverserSingleEntity(c *chk.C) { dstFileName := blobName scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) - // simulate cca with typical values - rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) - raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) - cca, err := raw.cook() - c.Assert(err, chk.IsNil) - // construct a local traverser - localTraverser := newLocalTraverser(&cca, false) + localTraverser := newLocalTraverser(filepath.Join(dstDirName, dstFileName), false, func() {}) // invoke the local traversal with a dummy processor localDummyProcessor := dummyProcessor{} - err = localTraverser.traverse(&localDummyProcessor, nil) + err := localTraverser.traverse(localDummyProcessor.process, nil) c.Assert(err, chk.IsNil) c.Assert(len(localDummyProcessor.record), chk.Equals, 1) // construct a blob traverser - blobTraverser, err := newBlobTraverser(&cca, true) - c.Assert(err, chk.IsNil) + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, false, func() {}) - // invoke the local traversal with a dummy processor + // invoke the blob traversal with a dummy processor blobDummyProcessor := dummyProcessor{} - err = blobTraverser.traverse(&blobDummyProcessor, nil) + err = blobTraverser.traverse(blobDummyProcessor.process, nil) c.Assert(err, chk.IsNil) c.Assert(len(blobDummyProcessor.record), chk.Equals, 1) @@ -69,7 +60,9 @@ func (s *syncTraverserSuite) TestSyncTraverserSingleEntity(c *chk.C) { } } -func (s *syncTraverserSuite) TestSyncTraverserContainerAndLocalDirectory(c *chk.C) { +// validate traversing a container and a local directory containing the same objects +// compare that blob and local traversers get consistent results +func (s *genericTraverserSuite) TestTraverserContainerAndLocalDirectory(c *chk.C) { bsu := getBSU() containerURL, containerName := createNewContainer(c, bsu) defer deleteContainer(c, containerURL) @@ -84,47 +77,45 @@ func (s *syncTraverserSuite) TestSyncTraverserContainerAndLocalDirectory(c *chk. // test two scenarios, either recursive or not for _, isRecursiveOn := range []bool{true, false} { - // simulate cca with typical values - rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) - raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) - raw.recursive = isRecursiveOn - cca, err := raw.cook() - c.Assert(err, chk.IsNil) - // construct a local traverser - localTraverser := newLocalTraverser(&cca, false) + localTraverser := newLocalTraverser(dstDirName, isRecursiveOn, func() {}) // invoke the local traversal with an indexer - localIndexer := newDestinationIndexer() - err = localTraverser.traverse(localIndexer, nil) + // so that the results are indexed for easy validation + localIndexer := newObjectIndexer() + err := localTraverser.traverse(localIndexer.store, nil) c.Assert(err, chk.IsNil) // construct a blob traverser - blobTraverser, err := newBlobTraverser(&cca, true) - c.Assert(err, chk.IsNil) + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + blobTraverser := newBlobTraverser(&rawContainerURLWithSAS, p, ctx, isRecursiveOn, func() {}) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} - err = blobTraverser.traverse(&blobDummyProcessor, nil) + err = blobTraverser.traverse(blobDummyProcessor.process, nil) c.Assert(err, chk.IsNil) // make sure the results are the same c.Assert(len(blobDummyProcessor.record), chk.Equals, len(localIndexer.indexMap)) - for _, entity := range blobDummyProcessor.record { - correspondingLocalFile, present := localIndexer.indexMap[entity.relativePath] + for _, storedObject := range blobDummyProcessor.record { + correspondingLocalFile, present := localIndexer.indexMap[storedObject.relativePath] c.Assert(present, chk.Equals, true) - c.Assert(correspondingLocalFile.name, chk.Equals, entity.name) - c.Assert(correspondingLocalFile.isMoreRecentThan(entity), chk.Equals, true) + c.Assert(correspondingLocalFile.name, chk.Equals, storedObject.name) + c.Assert(correspondingLocalFile.isMoreRecentThan(storedObject), chk.Equals, true) if !isRecursiveOn { - c.Assert(strings.Contains(entity.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + c.Assert(strings.Contains(storedObject.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) } } } } -func (s *syncTraverserSuite) TestSyncTraverserVirtualAndLocalDirectory(c *chk.C) { +// validate traversing a virtual and a local directory containing the same objects +// compare that blob and local traversers get consistent results +func (s *genericTraverserSuite) TestTraverserWithVirtualAndLocalDirectory(c *chk.C) { bsu := getBSU() containerURL, containerName := createNewContainer(c, bsu) defer deleteContainer(c, containerURL) @@ -140,41 +131,37 @@ func (s *syncTraverserSuite) TestSyncTraverserVirtualAndLocalDirectory(c *chk.C) // test two scenarios, either recursive or not for _, isRecursiveOn := range []bool{true, false} { - // simulate cca with typical values - rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) - raw := getDefaultRawInput(rawVirDirURLWithSAS.String(), path.Join(dstDirName, virDirName)) - raw.recursive = isRecursiveOn - cca, err := raw.cook() - c.Assert(err, chk.IsNil) - // construct a local traverser - localTraverser := newLocalTraverser(&cca, false) + localTraverser := newLocalTraverser(filepath.Join(dstDirName, virDirName), isRecursiveOn, func() {}) // invoke the local traversal with an indexer - localIndexer := newDestinationIndexer() - err = localTraverser.traverse(localIndexer, nil) + // so that the results are indexed for easy validation + localIndexer := newObjectIndexer() + err := localTraverser.traverse(localIndexer.store, nil) c.Assert(err, chk.IsNil) // construct a blob traverser - blobTraverser, err := newBlobTraverser(&cca, true) - c.Assert(err, chk.IsNil) + ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) + p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) + rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) + blobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, func() {}) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} - err = blobTraverser.traverse(&blobDummyProcessor, nil) + err = blobTraverser.traverse(blobDummyProcessor.process, nil) c.Assert(err, chk.IsNil) // make sure the results are the same c.Assert(len(blobDummyProcessor.record), chk.Equals, len(localIndexer.indexMap)) - for _, entity := range blobDummyProcessor.record { - correspondingLocalFile, present := localIndexer.indexMap[entity.relativePath] + for _, storedObject := range blobDummyProcessor.record { + correspondingLocalFile, present := localIndexer.indexMap[storedObject.relativePath] c.Assert(present, chk.Equals, true) - c.Assert(correspondingLocalFile.name, chk.Equals, entity.name) - c.Assert(correspondingLocalFile.isMoreRecentThan(entity), chk.Equals, true) + c.Assert(correspondingLocalFile.name, chk.Equals, storedObject.name) + c.Assert(correspondingLocalFile.isMoreRecentThan(storedObject), chk.Equals, true) if !isRecursiveOn { - c.Assert(strings.Contains(entity.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + c.Assert(strings.Contains(storedObject.relativePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) } } } diff --git a/cmd/zt_test_interceptor.go b/cmd/zt_interceptors_for_test.go similarity index 93% rename from cmd/zt_test_interceptor.go rename to cmd/zt_interceptors_for_test.go index 27a30b96c..51b49fb3b 100644 --- a/cmd/zt_test_interceptor.go +++ b/cmd/zt_interceptors_for_test.go @@ -69,3 +69,12 @@ func (mockedLifecycleManager) Error(string) func (mockedLifecycleManager) SurrenderControl() {} func (mockedLifecycleManager) InitiateProgressReporting(common.WorkController, bool) {} func (mockedLifecycleManager) GetEnvironmentVariable(common.EnvironmentVariable) string { return "" } + +type dummyProcessor struct { + record []storedObject +} + +func (d *dummyProcessor) process(storedObject storedObject) (err error) { + d.record = append(d.record, storedObject) + return +} diff --git a/cmd/zt_scenario_helpers_test.go b/cmd/zt_scenario_helpers_for_test.go similarity index 78% rename from cmd/zt_scenario_helpers_test.go rename to cmd/zt_scenario_helpers_for_test.go index b3e3f2865..9bfd9170c 100644 --- a/cmd/zt_scenario_helpers_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "github.com/Azure/azure-storage-blob-go/azblob" chk "gopkg.in/check.v1" "io/ioutil" @@ -36,12 +37,24 @@ func (scenarioHelper) generateFile(filePath string, fileSize int) ([]byte, error return bigBuff, err } -func (s scenarioHelper) generateRandomLocalFiles(c *chk.C, dirPath string, numOfFiles int) (fileList []string) { - for i := 0; i < numOfFiles; i++ { - fileName := filepath.Join(dirPath, generateName("random")) - fileList = append(fileList, fileName) +func (s scenarioHelper) generateRandomLocalFiles(c *chk.C, dirPath string, prefix string) (fileList []string) { + fileList = make([]string, 30) + for i := 0; i < 10; i++ { + fileName1 := generateName(prefix + "top") + fileName2 := generateName(prefix + "sub1/") + fileName3 := generateName(prefix + "sub2/") + + fileList[3*i] = fileName1 + fileList[3*i+1] = fileName2 + fileList[3*i+2] = fileName3 - _, err := s.generateFile(fileName, defaultFileSize) + _, err := s.generateFile(filepath.Join(dirPath, fileName1), defaultFileSize) + c.Assert(err, chk.IsNil) + + _, err = s.generateFile(filepath.Join(dirPath, fileName2), defaultFileSize) + c.Assert(err, chk.IsNil) + + _, err = s.generateFile(filepath.Join(dirPath, fileName3), defaultFileSize) c.Assert(err, chk.IsNil) } return @@ -108,3 +121,11 @@ func (scenarioHelper) getRawBlobURLWithSAS(c *chk.C, containerName string, blobN blobURLWithSAS := containerURLWithSAS.NewBlockBlobURL(blobName) return blobURLWithSAS.URL() } + +func (scenarioHelper) blobExists(blobURL azblob.BlobURL) bool { + _, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + if err == nil { + return true + } + return false +} diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 619b54131..7723d4010 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -101,7 +101,6 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { scenarioHelper{}.generateBlobs(c, containerURL, blobList) mockedRPC.reset() - // the file was created after the blob, so no sync should happen runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) @@ -388,7 +387,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithMissingDestination(c *chk.C) { } // there is a type mismatch between the source and destination -func (s *cmdIntegrationSuite) TestSyncDownloadWithContainerToFile(c *chk.C) { +func (s *cmdIntegrationSuite) TestSyncMismatchContainerAndFile(c *chk.C) { bsu := getBSU() // set up the container with numerous blobs @@ -419,10 +418,21 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithContainerToFile(c *chk.C) { // validate that the right number of transfers were scheduled c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) + + // reverse the source and destination + raw = getDefaultRawInput(filepath.Join(dstDirName, dstFileName), rawContainerURLWithSAS.String()) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) } // there is a type mismatch between the source and destination -func (s *cmdIntegrationSuite) TestSyncDownloadWithBlobToDirectory(c *chk.C) { +func (s *cmdIntegrationSuite) TestSyncMismatchBlobAndDirectory(c *chk.C) { bsu := getBSU() // set up the container with a single blob @@ -452,4 +462,15 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithBlobToDirectory(c *chk.C) { // validate that the right number of transfers were scheduled c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) + + // reverse the source and destination + raw = getDefaultRawInput(dstDirName, rawBlobURLWithSAS.String()) + + // type mismatch, we should get an error + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) } diff --git a/cmd/zt_sync_filter_test.go b/cmd/zt_sync_filter_test.go index b077c544d..e6fc84b6d 100644 --- a/cmd/zt_sync_filter_test.go +++ b/cmd/zt_sync_filter_test.go @@ -2,51 +2,76 @@ package cmd import ( chk "gopkg.in/check.v1" + "time" ) type syncFilterSuite struct{} var _ = chk.Suite(&syncFilterSuite{}) -func (s *syncFilterSuite) TestIncludeFilter(c *chk.C) { - // set up the filters - raw := rawSyncCmdArgs{} - includePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") - includeFilter := buildIncludeFilters(includePatternList)[0] - - // test the positive cases - filesToInclude := []string{"bla.pdf", "fancy.jpeg", "socool.jpeg.pdf", "exactName"} - for _, file := range filesToInclude { - passed := includeFilter.pass(genericEntity{name: file}) - c.Assert(passed, chk.Equals, true) - } - - // test the negative cases - notToInclude := []string{"bla.pdff", "fancyjpeg", "socool.jpeg.pdf.wut", "eexactName"} - for _, file := range notToInclude { - passed := includeFilter.pass(genericEntity{name: file}) - c.Assert(passed, chk.Equals, false) - } +func (s *syncFilterSuite) TestSyncSourceFilter(c *chk.C) { + // set up the indexer as well as the source filter + indexer := newObjectIndexer() + sourceFilter := newSyncSourceFilter(indexer) + + // create a sample destination object + sampleDestinationObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()} + + // test the filter in case a given source object is not present at the destination + // meaning no entry in the index, so the filter should pass the given object to schedule a transfer + passed := sourceFilter.doesPass(storedObject{name: "only_at_source", relativePath: "only_at_source", lastModifiedTime: time.Now()}) + c.Assert(passed, chk.Equals, true) + + // test the filter in case a given source object is present at the destination + // and it has a later modified time, so the filter should pass the give object to schedule a transfer + err := indexer.store(sampleDestinationObject) + c.Assert(err, chk.IsNil) + passed = sourceFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()}) + c.Assert(passed, chk.Equals, true) + + // test the filter in case a given source object is present at the destination + // but is has an earlier modified time compared to the one at the destination + // meaning that the source object is considered stale, so no transfer should be scheduled + err = indexer.store(sampleDestinationObject) + c.Assert(err, chk.IsNil) + passed = sourceFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour)}) + c.Assert(passed, chk.Equals, false) } -func (s *syncFilterSuite) TestExcludeFilter(c *chk.C) { - // set up the filters - raw := rawSyncCmdArgs{} - excludePatternList := raw.parsePatterns("*.pdf;*.jpeg;exactName") - excludeFilterList := buildExcludeFilters(excludePatternList) - - // test the positive cases - filesToPass := []string{"bla.pdfe", "fancy.jjpeg", "socool.png", "notexactName"} - for _, file := range filesToPass { - dummyProcessor := &dummyProcessor{} - passed := processIfPassedFilters(excludeFilterList, genericEntity{name: file}, dummyProcessor) - c.Assert(passed, chk.Equals, true) - } - - // test the negative cases - filesToNotPass := []string{"bla.pdff", "fancyjpeg", "socool.jpeg.pdf.wut", "eexactName"} - for _, file := range filesToNotPass { - passed := excludeFilter.pass(genericEntity{name: file}) - c.Assert(passed, chk.Equals, false) - } +func (s *syncFilterSuite) TestSyncDestinationFilter(c *chk.C) { + // set up the indexer as well as the destination filter + indexer := newObjectIndexer() + dummyProcessor := dummyProcessor{} + destinationFilter := newSyncDestinationFilter(indexer, dummyProcessor.process) + + // create a sample source object + sampleSourceObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()} + + // test the filter in case a given destination object is not present at the source + // meaning it is an extra file that needs to be deleted, so the filter should pass the given object to the recyclers + passed := destinationFilter.doesPass(storedObject{name: "only_at_source", relativePath: "only_at_source", lastModifiedTime: time.Now()}) + c.Assert(passed, chk.Equals, false) + c.Assert(len(dummyProcessor.record), chk.Equals, 1) + c.Assert(dummyProcessor.record[0].name, chk.Equals, "only_at_source") + + // reset dummy processor + dummyProcessor.record = make([]storedObject, 0) + + // test the filter in case a given destination object is present at the source + // and it has a later modified time, since the source data is stale, + // the filter should pass not the give object to schedule a transfer + err := indexer.store(sampleSourceObject) + c.Assert(err, chk.IsNil) + passed = destinationFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()}) + c.Assert(passed, chk.Equals, false) + c.Assert(len(dummyProcessor.record), chk.Equals, 0) + + // test the filter in case a given destination object is present at the source + // but is has an earlier modified time compared to the one at the source + // meaning that the source object should be transferred since the destination object is stale + err = indexer.store(sampleSourceObject) + c.Assert(err, chk.IsNil) + passed = destinationFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour)}) + c.Assert(passed, chk.Equals, true) + c.Assert(len(dummyProcessor.record), chk.Equals, 0) } diff --git a/cmd/zt_sync_processor_test.go b/cmd/zt_sync_processor_test.go index 6b217963c..233cebe25 100644 --- a/cmd/zt_sync_processor_test.go +++ b/cmd/zt_sync_processor_test.go @@ -1,131 +1,79 @@ package cmd import ( + "context" "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" chk "gopkg.in/check.v1" + "os" "path/filepath" - "time" ) type syncProcessorSuite struct{} var _ = chk.Suite(&syncProcessorSuite{}) -type syncProcessorSuiteHelper struct{} - -// return a list of sample entities -func (syncProcessorSuiteHelper) getSampleEntityList() []genericEntity { - return []genericEntity{ - {name: "file1", relativePath: "file1", lastModifiedTime: time.Now()}, - {name: "file2", relativePath: "file2", lastModifiedTime: time.Now()}, - {name: "file3", relativePath: "sub1/file3", lastModifiedTime: time.Now()}, - {name: "file4", relativePath: "sub1/file4", lastModifiedTime: time.Now()}, - {name: "file5", relativePath: "sub1/sub2/file5", lastModifiedTime: time.Now()}, - {name: "file6", relativePath: "sub1/sub2/file6", lastModifiedTime: time.Now()}, - } -} +func (s *syncProcessorSuite) TestLocalDeleter(c *chk.C) { + // set up the local file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := "extraFile.txt" + scenarioHelper{}.generateFilesFromList(c, dstDirName, []string{dstFileName}) -// given a list of entities, return the relative paths in a list, to help with validations -func (syncProcessorSuiteHelper) getExpectedTransferFromEntityList(entityList []genericEntity) []string { - expectedTransfers := make([]string, 0) - for _, entity := range entityList { - expectedTransfers = append(expectedTransfers, entity.relativePath) + // construct the cooked input to simulate user input + cca := &cookedSyncCmdArgs{ + destination: dstDirName, + force: true, } - return expectedTransfers -} - -func (s *syncProcessorSuite) TestSyncProcessorMultipleFiles(c *chk.C) { - bsu := getBSU() + // set up local deleter + deleter := newSyncLocalDeleteProcessor(cca) - // set up source and destination - containerURL, containerName := getContainerURL(c, bsu) - dstDirName := scenarioHelper{}.generateLocalDirectory(c) - - // set up interceptor - mockedRPC := interceptor{} - Rpc = mockedRPC.intercept - mockedRPC.init() - - // construct the raw input to simulate user input - rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) - raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) - cooked, err := raw.cook() + // validate that the file still exists + _, err := os.Stat(filepath.Join(dstDirName, dstFileName)) c.Assert(err, chk.IsNil) - // exercise the sync processor - sampleEntities := syncProcessorSuiteHelper{}.getSampleEntityList() - - for _, numOfParts := range []int{1, 3} { - // note we set the numOfTransfersPerPart here - syncProcessor := newSyncTransferProcessor(&cooked, len(sampleEntities)/numOfParts) - - // go through the entities and make sure they are processed without error - for _, entity := range sampleEntities { - err := syncProcessor.process(entity) - c.Assert(err, chk.IsNil) - } - - // make sure everything has been dispatched apart from the final one - c.Assert(syncProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(numOfParts-1)) - - // dispatch final part - jobInitiated, err := syncProcessor.dispatchFinalPart() - c.Assert(jobInitiated, chk.Equals, true) - c.Assert(err, chk.IsNil) - - // assert the right transfers were scheduled - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, - syncProcessorSuiteHelper{}.getExpectedTransferFromEntityList(sampleEntities), mockedRPC) + // exercise the deleter + err = deleter.removeImmediately(storedObject{relativePath: dstFileName}) + c.Assert(err, chk.IsNil) - mockedRPC.reset() - } + // validate that the file no longer exists + _, err = os.Stat(filepath.Join(dstDirName, dstFileName)) + c.Assert(err, chk.NotNil) } -func (s *syncProcessorSuite) TestSyncProcessorSingleFile(c *chk.C) { +func (s *syncProcessorSuite) TestBlobDeleter(c *chk.C) { bsu := getBSU() + blobName := "extraBlob.pdf" + + // set up the blob to delete containerURL, containerName := createNewContainer(c, bsu) defer deleteContainer(c, containerURL) + scenarioHelper{}.generateBlobs(c, containerURL, []string{blobName}) - // set up the container with a single blob - blobList := []string{"singlefile101"} - scenarioHelper{}.generateBlobs(c, containerURL, blobList) - c.Assert(containerURL, chk.NotNil) - - // set up the directory with a single file - dstDirName := scenarioHelper{}.generateLocalDirectory(c) - dstFileName := blobList[0] - scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) - - // set up interceptor - mockedRPC := interceptor{} - Rpc = mockedRPC.intercept - mockedRPC.init() - - // construct the raw input to simulate user input - rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) - raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) - cooked, err := raw.cook() + // validate that the blob exists + blobURL := containerURL.NewBlobURL(blobName) + _, err := blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) c.Assert(err, chk.IsNil) - // exercise the sync processor - syncProcessor := newSyncTransferProcessor(&cooked, 2) - entity := genericEntity{ - name: blobList[0], - relativePath: "", - lastModifiedTime: time.Now(), + // construct the cooked input to simulate user input + rawContainerURL := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + parts := azblob.NewBlobURLParts(rawContainerURL) + cca := &cookedSyncCmdArgs{ + destination: containerURL.String(), + destinationSAS: parts.SAS.Encode(), + credentialInfo: common.CredentialInfo{CredentialType: common.ECredentialType.Anonymous()}, + force: true, } - err = syncProcessor.process(entity) - c.Assert(err, chk.IsNil) - // no part should have been dispatched - c.Assert(syncProcessor.copyJobTemplate.PartNum, chk.Equals, common.PartNumber(0)) + // set up the blob deleter + deleter, err := newSyncBlobDeleteProcessor(cca) + c.Assert(err, chk.IsNil) - // dispatch final part - jobInitiated, err := syncProcessor.dispatchFinalPart() - c.Assert(jobInitiated, chk.Equals, true) + // exercise the deleter + err = deleter.removeImmediately(storedObject{relativePath: blobName}) + c.Assert(err, chk.IsNil) - // assert the right transfers were scheduled - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, - blobList, mockedRPC) + // validate that the blob was deleted + _, err = blobURL.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + c.Assert(err, chk.NotNil) } diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go new file mode 100644 index 000000000..5b7a7872a --- /dev/null +++ b/cmd/zt_sync_upload_test.go @@ -0,0 +1,333 @@ +package cmd + +import ( + "context" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" + chk "gopkg.in/check.v1" + "path/filepath" + "strings" + "time" +) + +// regular file->blob sync +func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { + bsu := getBSU() + + // set up the source as a single file + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + srcFileName := "singlefileisbest" + fileList := []string{srcFileName} + scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) + + // wait for 1 second so that the last modified time of the blob is guaranteed to be newer + time.Sleep(time.Second) + + // set up the destination container with a single blob + dstBlobName := srcFileName + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, []string{dstBlobName}) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dstBlobName) + raw := getDefaultRawInput(filepath.Join(srcDirName, srcFileName), rawBlobURLWithSAS.String()) + + // the blob was created after the file, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // sleep for 1 sec so the blob's last modified time is older + time.Sleep(time.Second) + + // recreate the file to have a later last modified time + scenarioHelper{}.generateFilesFromList(c, srcDirName, []string{srcFileName}) + mockedRPC.reset() + + // the file was created after the blob, so the sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) +} + +// regular directory->container sync but destination is empty, so everything has to be transferred +func (s *cmdIntegrationSuite) TestSyncUploadWithEmptyDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // set up an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(fileList)) + + // validate that the right transfers were sent + validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) + + // turn off recursive, this time only top blobs should be transferred + raw.recursive = false + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + c.Assert(len(mockedRPC.transfers), chk.Not(chk.Equals), len(fileList)) + + for _, transfer := range mockedRPC.transfers { + localRelativeFilePath := strings.Replace(transfer.Source, srcDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + c.Assert(strings.Contains(localRelativeFilePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + }) +} + +// regular directory->container sync but destination is identical to the source, transfers are scheduled based on lmt +func (s *cmdIntegrationSuite) TestSyncUploadWithIdenticalDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // set up an the container with the exact same files, but later lmts + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // wait for 1 second so that the last modified times of the blobs are guaranteed to be newer + time.Sleep(time.Second) + scenarioHelper{}.generateBlobs(c, containerURL, fileList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // wait for 1 second so that the last modified times of the files are guaranteed to be newer + time.Sleep(time.Second) + + // refresh the files' last modified time so that they are newer + scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) +} + +// regular container->directory sync where destination is missing some files from source, and also has some extra files +func (s *cmdIntegrationSuite) TestSyncUploadWithMismatchedDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // wait for 1 second so that the last modified times of the blobs are guaranteed to be newer + time.Sleep(time.Second) + + // set up an the container with half of the files, but later lmts + // also add some extra blobs that are not present at the source + extraBlobs := []string{"extraFile1.pdf, extraFile2.txt"} + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + scenarioHelper{}.generateBlobs(c, containerURL, fileList[0:len(fileList)/2]) + scenarioHelper{}.generateBlobs(c, containerURL, extraBlobs) + expectedOutput := fileList[len(fileList)/2:] + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, srcDirName, containerURL.String(), expectedOutput, mockedRPC) + + // make sure the extra blobs were deleted + for _, blobName := range extraBlobs { + exists := scenarioHelper{}.blobExists(containerURL.NewBlobURL(blobName)) + c.Assert(exists, chk.Equals, false) + } + }) +} + +// include flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncUploadWithIncludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // add special files that we wish to include + filesToInclude := []string{"important.pdf", "includeSub/amazing.jpeg", "exactName"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // set up the destination as an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + raw.include = includeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) + }) +} + +// exclude flag limits the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncUploadWithExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // add special files that we wish to exclude + filesToExclude := []string{"notGood.pdf", "excludeSub/lame.jpeg", "exactName"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToExclude) + excludeString := "*.pdf;*.jpeg;exactName" + + // set up the destination as an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) +} + +// include and exclude flag can work together to limit the scope of source/destination comparison +func (s *cmdIntegrationSuite) TestSyncUploadWithIncludeAndExcludeFlag(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // add special files that we wish to include + filesToInclude := []string{"important.pdf", "includeSub/amazing.jpeg"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToInclude) + includeString := "*.pdf;*.jpeg;exactName" + + // add special files that we wish to exclude + // note that the excluded files also match the include string + filesToExclude := []string{"sorry.pdf", "exclude/notGood.jpeg", "exactName", "sub/exactName"} + scenarioHelper{}.generateFilesFromList(c, srcDirName, filesToExclude) + excludeString := "so*;not*;exactName" + + // set up the destination as an empty container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + raw.include = includeString + raw.exclude = excludeString + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + validateTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) + }) +} + +// validate the bug fix for this scenario +func (s *cmdIntegrationSuite) TestSyncUploadWithMissingDestination(c *chk.C) { + bsu := getBSU() + + // set up the source with numerous files + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") + + // set up the destination as an non-existent container + containerURL, containerName := getContainerURL(c, bsu) + + // validate that the container does not exist + _, err := containerURL.GetProperties(context.Background(), azblob.LeaseAccessConditions{}) + c.Assert(err, chk.NotNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) + raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) + + runSyncAndVerify(c, raw, func(err error) { + // error should not be nil, but the app should not crash either + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} diff --git a/common/rpc-models.go b/common/rpc-models.go index 75b96d927..0cb94bbd2 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -69,31 +69,6 @@ type CredentialInfo struct { OAuthTokenInfo OAuthTokenInfo } -type SyncJobPartOrderRequest struct { - JobID JobID - FromTo FromTo - PartNumber PartNumber - LogLevel LogLevel - Include []string - Exclude []string - BlockSizeInBytes uint32 - SourceSAS string - DestinationSAS string - CopyJobRequest CopyJobPartOrderRequest - DeleteJobRequest CopyJobPartOrderRequest - // FilesDeletedLocally is used to keep track of the file that are deleted locally - // Since local files to delete are not sent as transfer to STE - // the count of the local files deletion is tracked using it. - FilesToDeleteLocally []string - // commandString hold the user given command which is logged to the Job log file - CommandString string - CredentialInfo CredentialInfo - - LocalFiles map[string]time.Time - - SourceFilesToExclude map[string]time.Time -} - type CopyJobPartOrderResponse struct { ErrorMsg string JobStarted bool From 895f169ea3958af1aa85dd3644452cf12e11a1b0 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Wed, 13 Feb 2019 01:11:10 -0800 Subject: [PATCH 14/64] Respect file scanning limit for sync --- cmd/sync.go | 1 - cmd/syncIndexer.go | 10 ++++++++++ cmd/syncTraverser.go | 5 +++-- cmd/zt_generic_traverser_test.go | 2 ++ cmd/zt_sync_download_test.go | 13 +++++++------ 5 files changed, 22 insertions(+), 9 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index 2a7359c32..63868b2b2 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -38,7 +38,6 @@ import ( "github.com/spf13/cobra" ) -// TODO plug this in // a max is set because we cannot buffer infinite amount of destination file info in memory const MaxNumberOfFilesAllowedInSync = 10000000 diff --git a/cmd/syncIndexer.go b/cmd/syncIndexer.go index 87ccd23d1..1d43d72ba 100644 --- a/cmd/syncIndexer.go +++ b/cmd/syncIndexer.go @@ -1,11 +1,16 @@ package cmd +import ( + "fmt" +) + // the objectIndexer is essential for the generic sync enumerator to work // it can serve as a: // 1. objectProcessor: accumulate a lookup map with given storedObjects // 2. resourceTraverser: go through the entities in the map like a traverser type objectIndexer struct { indexMap map[string]storedObject + counter int } func newObjectIndexer() *objectIndexer { @@ -17,7 +22,12 @@ func newObjectIndexer() *objectIndexer { // process the given stored object by indexing it using its relative path func (i *objectIndexer) store(storedObject storedObject) (err error) { + if i.counter == MaxNumberOfFilesAllowedInSync { + return fmt.Errorf("the maxium number of file allowed in sync is: %v", MaxNumberOfFilesAllowedInSync) + } + i.indexMap[storedObject.relativePath] = storedObject + i.counter += 1 return } diff --git a/cmd/syncTraverser.go b/cmd/syncTraverser.go index 0b9e9def9..4ceed265c 100644 --- a/cmd/syncTraverser.go +++ b/cmd/syncTraverser.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/Azure/azure-storage-azcopy/ste" "net/url" + "path" "strings" "sync/atomic" ) @@ -13,9 +14,9 @@ func newLocalTraverserForSync(cca *cookedSyncCmdArgs, isSource bool) (*localTrav var fullPath string if isSource { - fullPath = cca.source + fullPath = path.Clean(cca.source) } else { - fullPath = cca.destination + fullPath = path.Clean(cca.destination) } if strings.ContainsAny(fullPath, "*?") { diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go index 8b11b37e5..449340f46 100644 --- a/cmd/zt_generic_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -8,6 +8,7 @@ import ( chk "gopkg.in/check.v1" "path/filepath" "strings" + "time" ) type genericTraverserSuite struct{} @@ -72,6 +73,7 @@ func (s *genericTraverserSuite) TestTraverserContainerAndLocalDirectory(c *chk.C c.Assert(containerURL, chk.NotNil) // set up the destination with a folder that have the exact same files + time.Sleep(2 * time.Second) // make the lmts of local files newer dstDirName := scenarioHelper{}.generateLocalDirectory(c) scenarioHelper{}.generateFilesFromList(c, dstDirName, fileList) diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 7723d4010..4c1acfe3e 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -51,12 +51,13 @@ func validateTransfersAreScheduled(c *chk.C, srcDirName, dstDirName string, expe func getDefaultRawInput(src, dst string) rawSyncCmdArgs { return rawSyncCmdArgs{ - src: src, - dst: dst, - recursive: true, - logVerbosity: defaultLogVerbosityForSync, - output: defaultOutputFormatForSync, - force: true, + src: src, + dst: dst, + recursive: true, + logVerbosity: defaultLogVerbosityForSync, + output: defaultOutputFormatForSync, + force: true, + md5ValidationOption: common.DefaultHashValidationOption.String(), } } From 2ff766a5f5eb6b6f7c90afe233d202de33700819 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 19 Feb 2019 01:17:59 -0800 Subject: [PATCH 15/64] Added copyright headers to new files --- cmd/syncEnumerator.go | 28 +++++++ cmd/syncFilter.go | 56 ++++++++++---- cmd/syncIndexer.go | 25 +++++- cmd/syncProcessor.go | 20 +++++ cmd/syncTraverser.go | 20 +++++ cmd/zc_enumerator.go | 20 +++++ cmd/zc_filter.go | 20 +++++ cmd/zc_processor.go | 20 +++++ cmd/zc_traverser_blob.go | 29 +++++-- cmd/zc_traverser_local.go | 20 +++++ cmd/zt_generic_filter_test.go | 20 +++++ cmd/zt_generic_processor_test.go | 20 +++++ cmd/zt_generic_traverser_test.go | 20 +++++ cmd/zt_interceptors_for_test.go | 20 +++++ cmd/zt_scenario_helpers_for_test.go | 20 +++++ cmd/zt_sync_download_test.go | 115 ++++++++++++++++++++++++++++ cmd/zt_sync_filter_test.go | 22 +++++- cmd/zt_sync_processor_test.go | 20 +++++ cmd/zt_sync_upload_test.go | 20 +++++ cmd/zt_test.go | 20 +++++ 20 files changed, 528 insertions(+), 27 deletions(-) diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index d5b62a1cf..6e7a5e6ed 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( @@ -11,6 +31,7 @@ import ( // download implies transferring from a remote resource to the local disk // in this scenario, the destination is scanned/indexed first // then the source is scanned and filtered based on what the destination contains +// we do the local one first because it is assumed that local file systems will be faster to enumerate than remote resources func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { destinationTraverser, err := newLocalTraverserForSync(cca, false) if err != nil { @@ -47,6 +68,8 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat } // remove the extra files at the destination that were not present at the source + // we can only know what needs to be deleted when we have FINISHED traversing the remote source + // since only then can we know which local files definitely don't exist remotely deleteScheduler := newSyncLocalDeleteProcessor(cca) err = indexer.traverse(deleteScheduler.removeImmediately, nil) if err != nil { @@ -71,6 +94,7 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat // upload implies transferring from a local disk to a remote resource // in this scenario, the local disk (source) is scanned/indexed first // then the destination is scanned and filtered based on what the destination contains +// we do the local one first because it is assumed that local file systems will be faster to enumerate than remote resources func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator, err error) { sourceTraverser, err := newLocalTraverserForSync(cca, true) if err != nil { @@ -102,9 +126,13 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator if err != nil { return nil, fmt.Errorf("unable to instantiate destination cleaner due to: %s", err.Error()) } + // when uploading, we can delete remote objects immediately, because as we traverse the remote location + // we ALREADY have available a complete map of everything that exists locally + // so as soon as we see a remote destination object we can know whether it exists in the local source comparator := newSyncDestinationFilter(indexer, destinationCleaner.removeImmediately) finalize := func() error { + // schedule every local file that doesn't exist at the destination err = indexer.traverse(transferScheduler.scheduleCopyTransfer, filters) if err != nil { return err diff --git a/cmd/syncFilter.go b/cmd/syncFilter.go index 4df8533fa..149a16912 100644 --- a/cmd/syncFilter.go +++ b/cmd/syncFilter.go @@ -1,66 +1,88 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd // with the help of an objectIndexer containing the source objects // filter out the destination objects that should be transferred +// in other words, this should be used when destination is being enumerated secondly type syncDestinationFilter struct { - // the rejected objects would be passed to the recyclers - recyclers objectProcessor + // the rejected objects would be passed to the destinationCleaner + destinationCleaner objectProcessor // storing the source objects - i *objectIndexer + sourceIndex *objectIndexer } func newSyncDestinationFilter(i *objectIndexer, recyclers objectProcessor) objectFilter { - return &syncDestinationFilter{i: i, recyclers: recyclers} + return &syncDestinationFilter{sourceIndex: i, destinationCleaner: recyclers} } // it will only pass destination objects that are present in the indexer but stale compared to the entry in the map -// if the destinationObject is not present at all, it will be passed to the recyclers +// if the destinationObject is not present at all, it will be passed to the destinationCleaner // ex: we already know what the source contains, now we are looking at objects at the destination // if file x from the destination exists at the source, then we'd only transfer it if it is considered stale compared to its counterpart at the source // if file x does not exist at the source, then it is considered extra, and will be deleted func (f *syncDestinationFilter) doesPass(destinationObject storedObject) bool { - storedObjectInMap, present := f.i.indexMap[destinationObject.relativePath] + storedObjectInMap, present := f.sourceIndex.indexMap[destinationObject.relativePath] // if the destinationObject is present and stale, we let it pass if present { - defer delete(f.i.indexMap, destinationObject.relativePath) + defer delete(f.sourceIndex.indexMap, destinationObject.relativePath) if storedObjectInMap.isMoreRecentThan(destinationObject) { return true } - - return false + } else { + // purposefully ignore the error from destinationCleaner + // it's a tolerable error, since it just means some extra destination object might hang around a bit longer + _ = f.destinationCleaner(destinationObject) } - // purposefully ignore the error from recyclers - // it's a tolerable error, since it just means some extra destination object might hang around a bit longer - _ = f.recyclers(destinationObject) return false } // with the help of an objectIndexer containing the destination objects // filter out the source objects that should be transferred +// in other words, this should be used when source is being enumerated secondly type syncSourceFilter struct { // storing the destination objects - i *objectIndexer + destinationIndex *objectIndexer } func newSyncSourceFilter(i *objectIndexer) objectFilter { - return &syncSourceFilter{i: i} + return &syncSourceFilter{destinationIndex: i} } // it will only pass items that are: // 1. not present in the map // 2. present but is more recent than the entry in the map -// note: we remove the storedObject if it is present +// note: we remove the storedObject if it is present so that when we have finished +// the index will contain all objects which exist at the destination but were NOT passed to this routine func (f *syncSourceFilter) doesPass(sourceObject storedObject) bool { - storedObjectInMap, present := f.i.indexMap[sourceObject.relativePath] + storedObjectInMap, present := f.destinationIndex.indexMap[sourceObject.relativePath] // if the sourceObject is more recent, we let it pass if present { - defer delete(f.i.indexMap, sourceObject.relativePath) + defer delete(f.destinationIndex.indexMap, sourceObject.relativePath) if sourceObject.isMoreRecentThan(storedObjectInMap) { return true diff --git a/cmd/syncIndexer.go b/cmd/syncIndexer.go index 1d43d72ba..5d1e8313f 100644 --- a/cmd/syncIndexer.go +++ b/cmd/syncIndexer.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( @@ -14,10 +34,7 @@ type objectIndexer struct { } func newObjectIndexer() *objectIndexer { - indexer := objectIndexer{} - indexer.indexMap = make(map[string]storedObject) - - return &indexer + return &objectIndexer{indexMap: make(map[string]storedObject)} } // process the given stored object by indexing it using its relative path diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index 3317a6447..16ac9f388 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/syncTraverser.go b/cmd/syncTraverser.go index 4ceed265c..dd8bc8dd6 100644 --- a/cmd/syncTraverser.go +++ b/cmd/syncTraverser.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index 9bf6898c2..4a2516e2d 100644 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zc_filter.go b/cmd/zc_filter.go index 76b25e888..9418836f6 100644 --- a/cmd/zc_filter.go +++ b/cmd/zc_filter.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import "path" diff --git a/cmd/zc_processor.go b/cmd/zc_processor.go index 66b2d54b4..b5da778b3 100644 --- a/cmd/zc_processor.go +++ b/cmd/zc_processor.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index a1bebe732..b4e60f8d3 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( @@ -21,17 +41,16 @@ type blobTraverser struct { incrementEnumerationCounter func() } -func (t *blobTraverser) getPropertiesIfSingleBlob() (blobProps *azblob.BlobGetPropertiesResponse, isBlob bool) { +func (t *blobTraverser) getPropertiesIfSingleBlob() (*azblob.BlobGetPropertiesResponse, bool) { blobURL := azblob.NewBlobURL(*t.rawURL, t.p) blobProps, blobPropertiesErr := blobURL.GetProperties(t.ctx, azblob.BlobAccessConditions{}) // if there was no problem getting the properties, it means that we are looking at a single blob - if blobPropertiesErr == nil { - isBlob = true - return + if blobPropertiesErr == nil && !gCopyUtil.doesBlobRepresentAFolder(blobProps.NewMetadata()) { + return blobProps, true } - return + return nil, false } func (t *blobTraverser) traverse(processor objectProcessor, filters []objectFilter) (err error) { diff --git a/cmd/zc_traverser_local.go b/cmd/zc_traverser_local.go index 7745e2338..cbbb5230c 100644 --- a/cmd/zc_traverser_local.go +++ b/cmd/zc_traverser_local.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_generic_filter_test.go b/cmd/zt_generic_filter_test.go index 86903778f..5ff23f623 100644 --- a/cmd/zt_generic_filter_test.go +++ b/cmd/zt_generic_filter_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_generic_processor_test.go b/cmd/zt_generic_processor_test.go index b2c3b2cf9..9d072f482 100644 --- a/cmd/zt_generic_processor_test.go +++ b/cmd/zt_generic_processor_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go index 449340f46..ba3052705 100644 --- a/cmd/zt_generic_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_interceptors_for_test.go b/cmd/zt_interceptors_for_test.go index 51b49fb3b..d68c83a26 100644 --- a/cmd/zt_interceptors_for_test.go +++ b/cmd/zt_interceptors_for_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 9bfd9170c..2ad121d69 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 4c1acfe3e..c9a53d317 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -1,7 +1,30 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( + "bytes" + "context" "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/azblob" chk "gopkg.in/check.v1" "io/ioutil" "path/filepath" @@ -475,3 +498,95 @@ func (s *cmdIntegrationSuite) TestSyncMismatchBlobAndDirectory(c *chk.C) { c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) } + +// download a blob representing an ADLS directory to a local file +// we should recognize that there is a type mismatch +func (s *cmdIntegrationSuite) TestSyncDownloadADLSDirectoryTypeMismatch(c *chk.C) { + bsu := getBSU() + blobName := "adlsdir" + + // set up the destination as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobName + scenarioHelper{}.generateFilesFromList(c, dstDirName, []string{blobName}) + + // set up the container + containerURL, containerName := createNewContainer(c, bsu) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // create a single blob that represents an ADLS directory + _, err := containerURL.NewBlockBlobURL(blobName).Upload(context.Background(), bytes.NewReader(nil), + azblob.BlobHTTPHeaders{}, azblob.Metadata{"hdi_isfolder": "true"}, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobName) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + + // the file was created after the blob, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.NotNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) +} + +// adls directory -> local directory sync +// we should download every blob except the blob representing the directory +func (s *cmdIntegrationSuite) TestSyncDownloadWithADLSDirectory(c *chk.C) { + bsu := getBSU() + adlsDirName := "adlsdir" + + // set up the container with numerous blobs + containerURL, containerName := createNewContainer(c, bsu) + blobList := scenarioHelper{}.generateCommonRemoteScenario(c, containerURL, adlsDirName+"/") + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + c.Assert(len(blobList), chk.Not(chk.Equals), 0) + + // create a single blob that represents the ADLS directory + dirBlob := containerURL.NewBlockBlobURL(adlsDirName) + _, err := dirBlob.Upload(context.Background(), bytes.NewReader(nil), + azblob.BlobHTTPHeaders{}, azblob.Metadata{"hdi_isfolder": "true"}, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + + // set up the destination with an empty folder + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawContainerURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, adlsDirName) + raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(blobList)) + }) + + // turn off recursive, this time only top blobs should be transferred + raw.recursive = false + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + c.Assert(len(mockedRPC.transfers), chk.Not(chk.Equals), len(blobList)) + + for _, transfer := range mockedRPC.transfers { + localRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + c.Assert(strings.Contains(localRelativeFilePath, common.AZCOPY_PATH_SEPARATOR_STRING), chk.Equals, false) + } + }) +} diff --git a/cmd/zt_sync_filter_test.go b/cmd/zt_sync_filter_test.go index e6fc84b6d..e6890fb7d 100644 --- a/cmd/zt_sync_filter_test.go +++ b/cmd/zt_sync_filter_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( @@ -48,7 +68,7 @@ func (s *syncFilterSuite) TestSyncDestinationFilter(c *chk.C) { sampleSourceObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()} // test the filter in case a given destination object is not present at the source - // meaning it is an extra file that needs to be deleted, so the filter should pass the given object to the recyclers + // meaning it is an extra file that needs to be deleted, so the filter should pass the given object to the destinationCleaner passed := destinationFilter.doesPass(storedObject{name: "only_at_source", relativePath: "only_at_source", lastModifiedTime: time.Now()}) c.Assert(passed, chk.Equals, false) c.Assert(len(dummyProcessor.record), chk.Equals, 1) diff --git a/cmd/zt_sync_processor_test.go b/cmd/zt_sync_processor_test.go index 233cebe25..1dd02ea11 100644 --- a/cmd/zt_sync_processor_test.go +++ b/cmd/zt_sync_processor_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go index 5b7a7872a..2531a2bde 100644 --- a/cmd/zt_sync_upload_test.go +++ b/cmd/zt_sync_upload_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( diff --git a/cmd/zt_test.go b/cmd/zt_test.go index 10253880c..9c469b692 100644 --- a/cmd/zt_test.go +++ b/cmd/zt_test.go @@ -1,3 +1,23 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package cmd import ( From 522e210c097eb911d96cdfdc68b49e2a2d3c328f Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Tue, 19 Feb 2019 15:06:25 -0800 Subject: [PATCH 16/64] Minor fix to spacing in copy command's progress --- cmd/copy.go | 4 ++-- cmd/zt_sync_download_test.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index 5d75a49cf..dc21f4c91 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -805,7 +805,7 @@ func (cca *cookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { // display a scanning keyword if the job is not completely ordered var scanningString = "" if !summary.CompleteJobOrdered { - scanningString = "(scanning...)" + scanningString = " (scanning...)" } // compute the average throughput for the last time interval @@ -827,7 +827,7 @@ func (cca *cookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { summary.TotalTransfers, scanningString)) } else { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total %s, 2-sec Throughput (Mb/s): %v", + glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, 2-sec Throughput (Mb/s): %v", summary.TransfersCompleted, summary.TransfersFailed, summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index c9a53d317..384601959 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -557,6 +557,11 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithADLSDirectory(c *chk.C) { azblob.BlobHTTPHeaders{}, azblob.Metadata{"hdi_isfolder": "true"}, azblob.BlobAccessConditions{}) c.Assert(err, chk.IsNil) + // create an extra blob that represents an empty ADLS directory, which should never be picked up + _, err = containerURL.NewBlockBlobURL(adlsDirName+"/neverpickup").Upload(context.Background(), bytes.NewReader(nil), + azblob.BlobHTTPHeaders{}, azblob.Metadata{"hdi_isfolder": "true"}, azblob.BlobAccessConditions{}) + c.Assert(err, chk.IsNil) + // set up the destination with an empty folder dstDirName := scenarioHelper{}.generateLocalDirectory(c) From 1e4734849a4b63a9d2b6c762b4ccee0b68148f2f Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 14:49:36 +1300 Subject: [PATCH 17/64] Update to 2018-11-09 Swagger schema This is purely update the schema file and regen. Not everything compiles yet --- azbfs/azure_dfs_swagger.json | 3645 ++++++++++++------------ azbfs/zz_generated_client.go | 1276 +-------- azbfs/zz_generated_filesystem.go | 453 +++ azbfs/zz_generated_models.go | 966 ++++--- azbfs/zz_generated_path.go | 1009 +++++++ azbfs/zz_generated_responder_policy.go | 2 +- azbfs/zz_generated_version.go | 2 +- 7 files changed, 3905 insertions(+), 3448 deletions(-) create mode 100644 azbfs/zz_generated_filesystem.go create mode 100644 azbfs/zz_generated_path.go diff --git a/azbfs/azure_dfs_swagger.json b/azbfs/azure_dfs_swagger.json index a8bcfc33e..1a8a64dd4 100644 --- a/azbfs/azure_dfs_swagger.json +++ b/azbfs/azure_dfs_swagger.json @@ -1,1760 +1,1885 @@ -{ - "swagger": "2.0", - "info": { - "description": "Azure Data Lake Storage provides storage for Hadoop and other big data workloads.", - "title": "Azure Data Lake Storage REST API", - "version": "2018-06-17" - }, - "x-ms-parameterized-host": { - "hostTemplate": "{accountName}.{dnsSuffix}", - "parameters": [ - { - "$ref": "#/parameters/accountName" - }, - { - "$ref": "#/parameters/dnsSuffix" - } - ] - }, - "schemes": [ - "http", - "https" - ], - "produces": [ - "application/json" - ], - "tags": [ - { - "name": "Account Operations" - }, - { - "name": "Filesystem Operations" - }, - { - "name": "File and Directory Operations" - } - ], - "parameters": { - "Version": { - "description": "Specifies the version of the REST protocol used for processing the request. This is required when using shared key authorization.", - "in": "header", - "name": "x-ms-version", - "required": true, - "type": "string" - }, - "accountName": { - "description": "The Azure Storage account name.", - "in": "path", - "name": "accountName", - "required": true, - "type": "string", - "x-ms-skip-url-encoding": true - }, - "dnsSuffix": { - "default": "dfs.core.windows.net", - "description": "The DNS suffix for the Azure Data Lake Storage endpoint.", - "in": "path", - "name": "dnsSuffix", - "required": true, - "type": "string", - "x-ms-skip-url-encoding": true - } - }, - "definitions": { - "ErrorSchema": { - "properties": { - "error": { - "description": "The service error response object.", - "properties": { - "code": { - "description": "The service error code.", - "type": "string" - }, - "message": { - "description": "The service error message.", - "type": "string" - } - } - } - } - }, - "ListEntrySchema": { - "properties": { - "name": { - "type": "string" - }, - "isDirectory": { - "default": false, - "type": "boolean" - }, - "lastModified": { - "type": "string" - }, - "eTag": { - "type": "string" - }, - "contentLength": { - "type": "integer", - "format": "int64" - }, - "owner": { - "type": "string" - }, - "group": { - "type": "string" - }, - "permissions": { - "type": "string" - } - } - }, - "ListSchema": { - "properties": { - "paths": { - "type": "array", - "items": { - "$ref": "#/definitions/ListEntrySchema" - } - } - } - }, - "ListFilesystemEntry": { - "properties": { - "name": { - "type": "string" - }, - "lastModified": { - "type": "string" - }, - "eTag": { - "type": "string" - } - } - }, - "ListFilesystemSchema": { - "properties": { - "filesystems": { - "type": "array", - "items": { - "$ref": "#/definitions/ListFilesystemEntry" - } - } - } - } - }, - "responses": { - "ErrorResponse": { - "description": "An error occurred. The possible HTTP status, code, and message strings are listed below:\n* 400 Bad Request, ContentLengthMustBeZero, \"The Content-Length request header must be zero.\"\n* 400 Bad Request, InvalidAuthenticationInfo, \"Authentication information is not given in the correct format. Check the value of Authorization header.\"\n* 400 Bad Request, InvalidFlushPosition, \"The uploaded data is not contiguous or the position query parameter value is not equal to the length of the file after appending the uploaded data.\"\n* 400 Bad Request, InvalidHeaderValue, \"The value for one of the HTTP headers is not in the correct format.\"\n* 400 Bad Request, InvalidHttpVerb, \"The HTTP verb specified is invalid - it is not recognized by the server.\"\n* 400 Bad Request, InvalidInput, \"One of the request inputs is not valid.\"\n* 400 Bad Request, InvalidPropertyName, \"A property name cannot be empty.\"\n* 400 Bad Request, InvalidPropertyName, \"The property name contains invalid characters.\"\n* 400 Bad Request, InvalidQueryParameterValue, \"Value for one of the query parameters specified in the request URI is invalid.\"\n* 400 Bad Request, InvalidResourceName, \"The specifed resource name contains invalid characters.\"\n* 400 Bad Request, InvalidSourceUri, \"The source URI is invalid.\"\n* 400 Bad Request, InvalidUri, \"The request URI is invalid.\"\n* 400 Bad Request, MissingRequiredHeader, \"An HTTP header that's mandatory for this request is not specified.\"\n* 400 Bad Request, MissingRequiredQueryParameter, \"A query parameter that's mandatory for this request is not specified.\"\n* 400 Bad Request, MultipleConditionHeadersNotSupported, \"Multiple condition headers are not supported.\"\n* 400 Bad Request, OutOfRangeInput, \"One of the request inputs is out of range.\"\n* 400 Bad Request, OutOfRangeQueryParameterValue, \"One of the query parameters specified in the request URI is outside the permissible range.\"\n* 400 Bad Request, UnsupportedHeader, \"One of the headers specified in the request is not supported.\"\n* 400 Bad Request, UnsupportedQueryParameter, \"One of the query parameters specified in the request URI is not supported.\"\n* 400 Bad Request, UnsupportedRestVersion, \"The specified Rest Version is Unsupported.\"\n* 403 Forbidden, AccountIsDisabled, \"The specified account is disabled.\"\n* 403 Forbidden, AuthorizationFailure, \"This request is not authorized to perform this operation.\"\n* 403 Forbidden, InsufficientAccountPermissions, \"The account being accessed does not have sufficient permissions to execute this operation.\"\n* 404 Not Found, FilesystemNotFound, \"The specified filesystem does not exist.\"\n* 404 Not Found, PathNotFound, \"The specified path does not exist.\"\n* 404 Not Found, RenameDestinationParentPathNotFound, \"The parent directory of the destination path does not exist.\"\n* 404 Not Found, ResourceNotFound, \"The specified resource does not exist.\"\n* 404 Not Found, SourcePathNotFound, \"The source path for a rename operation does not exist.\"\n* 405 Method Not Allowed, UnsupportedHttpVerb, \"The resource doesn't support the specified HTTP verb.\"\n* 409 Conflict, DestinationPathIsBeingDeleted, \"The specified destination path is marked to be deleted.\"\n* 409 Conflict, DirectoryNotEmpty, \"The recursive query parameter value must be true to delete a non-empty directory.\"\n* 409 Conflict, FilesystemAlreadyExists, \"The specified filesystem already exists.\"\n* 409 Conflict, FilesystemBeingDeleted, \"The specified filesystem is being deleted.\"\n* 409 Conflict, InvalidDestinationPath, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"* 409 Conflict, InvalidFlushOperation, \"The resource was created or modified by the Blob Service API and cannot be written to by the Data Lake Storage Service API.\"\n* 409 Conflict, InvalidRenameSourcePath, \"The source directory cannot be the same as the destination directory, nor can the destination be a subdirectory of the source directory.\"\n* 409 Conflict, InvalidSourceOrDestinationResourceType, \"The source and destination resource type must be identical.\"\n* 409 Conflict, LeaseAlreadyPresent, \"There is already a lease present.\"\n* 409 Conflict, LeaseIdMismatchWithLeaseOperation, \"The lease ID specified did not match the lease ID for the resource with the specified lease operation.\"\n* 409 Conflict, LeaseIsAlreadyBroken, \"The lease has already been broken and cannot be broken again.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeAcquired, \"The lease ID matched, but the lease is currently in breaking state and cannot be acquired until it is broken.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeChanged, \"The lease ID matched, but the lease is currently in breaking state and cannot be changed.\"\n* 409 Conflict, LeaseIsBrokenAndCannotBeRenewed, \"The lease ID matched, but the lease has been broken explicitly and cannot be renewed.\"\n* 409 Conflict, LeaseNameMismatch, \"The lease name specified did not match the existing lease name.\"\n* 409 Conflict, LeaseNotPresentWithLeaseOperation, \"The lease ID is not present with the specified lease operation.\"\n* 409 Conflict, PathAlreadyExists, \"The specified path already exists.\"\n* 409 Conflict, PathConflict, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"\n* 409 Conflict, SourcePathIsBeingDeleted, \"The specified source path is marked to be deleted.\"\n* 409 Conflict, ResourceTypeMismatch, \"The resource type specified in the request does not match the type of the resource.\"\n* 412 Precondition Failed, ConditionNotMet, \"The condition specified using HTTP conditional header(s) is not met.\"\n* 412 Precondition Failed, LeaseIdMismatch, \"The lease ID specified did not match the lease ID for the resource.\"\n* 412 Precondition Failed, LeaseIdMissing, \"There is currently a lease on the resource and no lease ID was specified in the request.\"\n* 412 Precondition Failed, LeaseNotPresent, \"There is currently no lease on the resource.\"\n* 412 Precondition Failed, LeaseLost, \"A lease ID was specified, but the lease for the resource has expired.\"\n* 412 Precondition Failed, SourceConditionNotMet, \"The source condition specified using HTTP conditional header(s) is not met.\"\n* 413 Request Entity Too Large, RequestBodyTooLarge, \"The request body is too large and exceeds the maximum permissible limit.\"\n* 416 Requested Range Not Satisfiable, InvalidRange, \"The range specified is invalid for the current size of the resource.\"\n* 500 Internal Server Error, InternalError, \"The server encountered an internal error. Please retry the request.\"\n* 500 Internal Server Error, OperationTimedOut, \"The operation could not be completed within the permitted time.\"\n* 503 Service Unavailable, ServerBusy, \"Egress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Ingress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Operations per second is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"The server is currently unable to receive requests. Please retry your request.\"", - "headers": { - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - }, - "schema": { - "$ref": "#/definitions/ErrorSchema" - } - } - }, - "paths": { - "/": { - "get": { - "operationId": "ListFilesystems", - "summary": "List Filesystems", - "description": "List filesystems and their properties in given account.", - "tags": [ - "Account Operations" - ], - "responses": { - "200": { - "description": "OK", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "If the number of filesystems to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", - "type": "string" - }, - "Content-Type": { - "description": "The content type of list filesystem response. The default content type is application/json.", - "type": "string" - } - }, - "schema": { - "$ref": "#/definitions/ListFilesystemSchema" - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "resource", - "in": "query", - "description": "The value must be \"account\" for all account operations.", - "required": true, - "type": "string" - }, - { - "name": "prefix", - "in": "query", - "description": "Filters results to filesystems within the specified prefix.", - "required": false, - "type": "string" - }, - { - "name": "continuation", - "in": "query", - "description": "The number of filesystems returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", - "required": false, - "type": "string" - }, - { - "name": "maxResults", - "in": "query", - "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-client-request-id", - "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", - "in": "header", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "timeout", - "in": "query", - "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-date", - "in": "header", - "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", - "required": false, - "type": "string" - }, - { - "$ref": "#/parameters/Version" - } - ] - } - }, - "/{filesystem}": { - "put": { - "operationId": "CreateFilesystem", - "summary": "Create Filesystem", - "description": "Create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. This operation does not support conditional HTTP requests.", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "201": { - "description": "Created", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Operations on files and directories do not affect the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-namespace-enabled": { - "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "x-ms-properties", - "description": "User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "patch": { - "operationId": "SetFilesystemProperties", - "summary": "Set Filesystem Properties", - "description": "Set properties for the filesystem. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "x-ms-properties", - "description": "Optional. User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded. If the filesystem exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "get": { - "operationId": "ListPaths", - "summary": "List Paths", - "description": "List filesystem paths and their properties.", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "If the number of paths to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", - "type": "string" - } - }, - "schema": { - "$ref": "#/definitions/ListSchema" - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "directory", - "in": "query", - "description": "Filters results to paths within the specified directory. An error occurs if the directory does not exist.", - "required": false, - "type": "string" - }, - { - "name": "recursive", - "in": "query", - "description": "If \"true\", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If \"directory\" is specified, the list will only include paths that share the same root.", - "required": true, - "type": "boolean" - }, - { - "name": "continuation", - "in": "query", - "description": "The number of paths returned with each invocation is limited. If the number of paths to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", - "required": false, - "type": "string" - }, - { - "name": "maxResults", - "in": "query", - "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - } - ] - }, - "head": { - "operationId": "GetFilesystemProperties", - "summary": "Get Filesystem Properties.", - "description": "All system and user-defined filesystem properties are specified in the response headers.", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the filesystem. A comma-separated list of name and value pairs in the format \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-namespace-enabled": { - "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - } - }, - "delete": { - "operationId": "DeleteFilesystem", - "summary": "Delete Filesystem", - "description": "Marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional error information indicating that the filesystem is being deleted. All other operations, including operations on any files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "Filesystem Operations" - ], - "responses": { - "202": { - "description": "Accepted", - "headers": { - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "parameters": [ - { - "name": "filesystem", - "in": "path", - "description": "The filesystem identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have between 3 and 63 characters.", - "required": true, - "type": "string" - }, - { - "name": "resource", - "in": "query", - "description": "The value must be \"filesystem\" for all filesystem operations.", - "required": true, - "type": "string" - }, - { - "name": "x-ms-client-request-id", - "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", - "in": "header", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "timeout", - "in": "query", - "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-date", - "in": "header", - "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", - "required": false, - "type": "string" - }, - { - "$ref": "#/parameters/Version" - } - ] - }, - "/{filesystem}/{path}": { - "put": { - "operationId": "CreatePath", - "summary": "Create File | Create Directory | Rename File | Rename Directory", - "description": "Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: \"*\".", - "consumes": [ - "application/octet-stream" - ], - "tags": [ - "File and Directory Operations" - ], - "responses": { - "201": { - "description": "The file or directory was created.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "resource", - "in": "query", - "description": "Required only for Create File and Create Directory. The value must be \"file\" or \"directory\".", - "required": false, - "type": "string" - }, - { - "name": "continuation", - "in": "query", - "description": "Optional. When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", - "required": false, - "type": "string" - }, - { - "name": "mode", - "in": "query", - "description": "Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be \"legacy\" or \"posix\", and the default value will be \"posix\". ", - "required": false, - "type": "string" - }, - { - "name": "Cache-Control", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "Content-Encoding", - "in": "header", - "description": "Optional. Specifies which content encodings have been applied to the file. This value is returned to the client when the \"Read File\" operation is performed.", - "required": false, - "type": "string" - }, - { - "name": "Content-Language", - "in": "header", - "description": "Optional. Specifies the natural language used by the intended audience for the file.", - "required": false, - "type": "string" - }, - { - "name": "Content-Disposition", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-cache-control", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-type", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-encoding", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-language", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-disposition", - "in": "header", - "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-rename-source", - "in": "header", - "description": "An optional file or directory to be renamed. The value must have the following format: \"/{filesysystem}/{path}\". If \"x-ms-properties\" is specified, the properties will overwrite the existing properties; otherwise, the existing properties will be preserved.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "Optional. A lease ID for the path specified in the URI. The path to be overwritten must have an active lease and the lease ID must match.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-proposed-lease-id", - "in": "header", - "description": "Optional for create operations. Required when \"x-ms-lease-action\" is used. A lease will be acquired using the proposed ID when the resource is created.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-lease-id", - "in": "header", - "description": "Optional for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID must match.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-properties", - "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-permissions", - "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-match", - "description": "Optional. An ETag value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-none-match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-modified-since", - "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-source-if-unmodified-since", - "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "patch": { - "operationId": "UpdatePath", - "summary": "Append Data | Flush Data | Set Properties | Set Access Control", - "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "consumes": [ - "application/octet-stream", - "text/plain" - ], - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "The data was flushed (written) to the file or the properties were set successfully.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "x-ms-properties": { - "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - } - }, - "202": { - "description": "The uploaded data was accepted.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "action", - "in": "query", - "description": "The action must be \"append\" to upload data to be appended to a file, \"flush\" to flush previously uploaded data to a file, \"setProperties\" to set the properties of a file or directory, or \"setAccessControl\" to set the owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are mutually exclusive.", - "required": true, - "type": "string" - }, - { - "name": "position", - "in": "query", - "description": "This parameter allows the caller to upload data in parallel and control the order in which it is appended to the file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to the file. The value must be the position where the data is to be appended. Uploaded data is not immediately flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter must be specified and equal to the length of the file after all data has been written, and there must not be a request entity body included with the request.", - "format": "int64", - "required": false, - "type": "integer" - }, - { - "name": "retainUncommittedData", - "in": "query", - "description": "Valid only for flush operations. If \"true\", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after the flush operation. The default is false. Data at offsets less than the specified position are written to the file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future flush operation.", - "required": false, - "type": "boolean" - }, - { - "name": "Content-Length", - "in": "header", - "description": "Required for \"Append Data\" and \"Flush Data\". Must be 0 for \"Flush Data\". Must be the length of the request content in bytes for \"Append Data\".", - "minimum": 0, - "required": false, - "type": "string" - }, - { - "name": "x-ms-lease-action", - "in": "header", - "description": "Optional. The lease action can be \"renew\" to renew an existing lease or \"release\" to release a lease.", - "type": "string" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "The lease ID must be specified if there is an active lease.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-cache-control", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-type", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-disposition", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-encoding", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-content-language", - "in": "header", - "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-properties", - "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded. Valid only for the setProperties operation. If the file or directory exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-owner", - "description": "Optional and valid only for the setAccessControl operation. Sets the owner of the file or directory.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-group", - "description": "Optional and valid only for the setAccessControl operation. Sets the owning group of the file or directory.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-permissions", - "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction with x-ms-acl.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-ms-acl", - "description": "Optional and valid only for the setAccessControl operation. Sets POSIX access control rights on files and directories. The value is a comma-separated list of access control entries that fully replaces the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or group identifier, and permissions in the format \"[scope:][type]:[id]:[permissions]\". The scope must be \"default\" to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL. There are four ACE types: \"user\" grants rights to the owner or a named user, \"group\" grants rights to the owning group or a named group, \"mask\" restricts rights granted to named users and the members of groups, and \"other\" grants rights to all users not found in any of the other entries. The user or group identifier is omitted for entries of type \"mask\" and \"other\". The user or group identifier is also omitted for the owner and owning group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If access is not granted, the '-' character is used to denote that the permission is denied. For example, the following ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning group, and nothing to everyone else: \"user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx\". Invalid in conjunction with x-ms-permissions.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "x-http-method-override", - "description": "Optional. Override the http verb on the service side. Some older http clients do not support PATCH", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "requestBody", - "description": "Valid only for append operations. The data to be uploaded and appended to the file.", - "in": "body", - "required": false, - "schema": { - "type": "object", - "format": "file" - } - } - ] - }, - "post": { - "operationId": "LeasePath", - "summary": "Lease Path", - "description": "Create and manage a lease to restrict write and delete access to the path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "The \"renew\", \"change\" or \"release\" action was successful.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file was last modified. Write operations on the file update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-lease-id": { - "description": "A successful \"renew\" action returns the lease ID.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - } - } - }, - "201": { - "description": "A new lease has been created. The \"acquire\" action was successful.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-lease-id": { - "description": "A successful \"acquire\" action returns the lease ID.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - } - } - }, - "202": { - "description": "The \"break\" lease action was successful.", - "headers": { - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-lease-time": { - "description": "The time remaining in the lease period in seconds.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "x-ms-lease-action", - "in": "header", - "description": "There are five lease actions: \"acquire\", \"break\", \"change\", \"renew\", and \"release\". Use \"acquire\" and specify the \"x-ms-proposed-lease-id\" and \"x-ms-lease-duration\" to acquire a new lease. Use \"break\" to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease operation except break and release can be performed on the file. When a lease is successfully broken, the response indicates the interval in seconds until a new lease can be acquired. Use \"change\" and specify the current lease ID in \"x-ms-lease-id\" and the new lease ID in \"x-ms-proposed-lease-id\" to change the lease ID of an active lease. Use \"renew\" and specify the \"x-ms-lease-id\" to renew an existing lease. Use \"release\" and specify the \"x-ms-lease-id\" to release a lease.", - "required": true, - "type": "string" - }, - { - "name": "x-ms-lease-duration", - "in": "header", - "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", - "format": "int32", - "required": false, - "type": "integer" - }, - { - "name": "x-ms-lease-break-period", - "in": "header", - "description": "The lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. The lease break duration must be between 0 and 60 seconds.", - "format": "int32", - "required": false, - "type": "integer" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "Required when \"x-ms-lease-action\" is \"renew\", \"change\" or \"release\". For the renew and release actions, this must match the current lease ID.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "x-ms-proposed-lease-id", - "in": "header", - "description": "Required when \"x-ms-lease-action\" is \"acquire\" or \"change\". A lease will be acquired with this lease ID if the operation is successful.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "get": { - "operationId": "ReadPath", - "summary": "Read File", - "description": "Read the contents of a file. For read operations, range requests are supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "produces": [ - "application/json", - "application/octet-stream", - "text/plain" - ], - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "Ok", - "headers": { - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-resource-type": { - "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-lease-duration": { - "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", - "type": "string" - }, - "x-ms-lease-state": { - "description": "Lease state of the resource. ", - "type": "string" - }, - "x-ms-lease-status": { - "description": "The lease status of the resource.", - "type": "string" - } - }, - "schema": { - "type": "file" - } - }, - "206": { - "description": "Partial content", - "headers": { - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-resource-type": { - "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-lease-duration": { - "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", - "type": "string" - }, - "x-ms-lease-state": { - "description": "Lease state of the resource. ", - "type": "string" - }, - "x-ms-lease-status": { - "description": "The lease status of the resource.", - "type": "string" - } - }, - "schema": { - "type": "file" - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "in": "header", - "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved.", - "required": false, - "type": "string", - "name": "Range" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "head": { - "operationId": "GetPathProperties", - "summary": "Get Properties | Get Access Control List", - "description": "Get the properties for a file or directory, and optionally include the access control list. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "Returns all properties for the file or directory.", - "headers": { - "Accept-Ranges": { - "description": "Indicates that the service supports requests for partial file content.", - "type": "string" - }, - "Cache-Control": { - "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Disposition": { - "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Encoding": { - "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Language": { - "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", - "type": "string" - }, - "Content-Length": { - "description": "The size of the resource in bytes.", - "type": "string" - }, - "Content-Range": { - "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", - "type": "string" - }, - "Content-Type": { - "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", - "type": "string" - }, - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "ETag": { - "description": "An HTTP entity tag associated with the file or directory.", - "type": "string" - }, - "Last-Modified": { - "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-resource-type": { - "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", - "type": "string" - }, - "x-ms-properties": { - "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is base64 encoded.", - "type": "string" - }, - "x-ms-owner": { - "description": "The owner of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-group": { - "description": "The owning group of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-permissions": { - "description": "The POSIX access permissions for the file owner, the file owning group, and others. Included in the response if Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-acl": { - "description": "The POSIX access control list for the file or directory. Included in the response only if the action is \"getAccessControl\" and Hierarchical Namespace is enabled for the account.", - "type": "string" - }, - "x-ms-lease-duration": { - "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", - "type": "string" - }, - "x-ms-lease-state": { - "description": "Lease state of the resource. ", - "type": "string" - }, - "x-ms-lease-status": { - "description": "The lease status of the resource.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "action", - "in": "query", - "description": "Optional. If the value is \"getAccessControl\" the access control list is returned in the response headers (Hierarchical Namespace must be enabled for the account).", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "delete": { - "operationId": "DeletePath", - "summary": "Delete File | Delete Directory", - "description": "Delete the file or directory. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", - "tags": [ - "File and Directory Operations" - ], - "responses": { - "200": { - "description": "The file was deleted.", - "headers": { - "Date": { - "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", - "type": "string" - }, - "x-ms-request-id": { - "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "type": "string" - }, - "x-ms-version": { - "description": "The version of the REST protocol used to process the request.", - "type": "string" - }, - "x-ms-continuation": { - "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", - "type": "string" - } - } - }, - "default": { - "$ref": "#/responses/ErrorResponse" - } - }, - "parameters": [ - { - "name": "recursive", - "in": "query", - "description": "Required and valid only when the resource is a directory. If \"true\", all paths beneath the directory will be deleted. If \"false\" and the directory is non-empty, an error occurs.", - "required": false, - "type": "boolean" - }, - { - "name": "continuation", - "in": "query", - "description": "Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", - "required": false, - "type": "string" - }, - { - "name": "x-ms-lease-id", - "in": "header", - "description": "The lease ID must be specified if there is an active lease.", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "If-Match", - "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-None-Match", - "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Modified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - }, - { - "name": "If-Unmodified-Since", - "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", - "in": "header", - "required": false, - "type": "string" - } - ] - }, - "parameters": [ - { - "name": "filesystem", - "in": "path", - "description": "The filesystem identifier.", - "minLength": 3, - "maxLength": 63, - "required": true, - "type": "string" - }, - { - "name": "path", - "in": "path", - "description": "The file or directory path.", - "required": true, - "type": "string" - }, - { - "name": "x-ms-client-request-id", - "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", - "in": "header", - "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", - "required": false, - "type": "string" - }, - { - "name": "timeout", - "in": "query", - "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", - "format": "int32", - "minimum": 1, - "required": false, - "type": "integer" - }, - { - "name": "x-ms-date", - "in": "header", - "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", - "required": false, - "type": "string" - }, - { - "$ref": "#/parameters/Version" - } - ] - } - } -} +{ + "swagger": "2.0", + "info": { + "description": "Azure Data Lake Storage provides storage for Hadoop and other big data workloads.", + "title": "Azure Data Lake Storage REST API", + "version": "2018-11-09", + "x-ms-code-generation-settings": { + "internalConstructors": true, + "name": "DataLakeStorageClient" + } + }, + "x-ms-parameterized-host": { + "hostTemplate": "{accountName}.{dnsSuffix}", + "parameters": [ + { + "$ref": "#/parameters/accountName" + }, + { + "$ref": "#/parameters/dnsSuffix" + } + ] + }, + "schemes": [ + "http", + "https" + ], + "produces": [ + "application/json" + ], + "tags": [ + { + "name": "Account Operations" + }, + { + "name": "Filesystem Operations" + }, + { + "name": "File and Directory Operations" + } + ], + "parameters": { + "Version": { + "description": "Specifies the version of the REST protocol used for processing the request. This is required when using shared key authorization.", + "in": "header", + "name": "x-ms-version", + "required": false, + "type": "string", + "x-ms-parameter-location": "client" + }, + "accountName": { + "description": "The Azure Storage account name.", + "in": "path", + "name": "accountName", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + }, + "dnsSuffix": { + "default": "dfs.core.windows.net", + "description": "The DNS suffix for the Azure Data Lake Storage endpoint.", + "in": "path", + "name": "dnsSuffix", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + } + }, + "definitions": { + "DataLakeStorageError": { + "properties": { + "error": { + "description": "The service error response object.", + "properties": { + "code": { + "description": "The service error code.", + "type": "string" + }, + "message": { + "description": "The service error message.", + "type": "string" + } + } + } + } + }, + "Path": { + "properties": { + "name": { + "type": "string" + }, + "isDirectory": { + "default": false, + "type": "boolean" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + }, + "contentLength": { + "type": "integer", + "format": "int64" + }, + "owner": { + "type": "string" + }, + "group": { + "type": "string" + }, + "permissions": { + "type": "string" + } + } + }, + "PathList": { + "properties": { + "paths": { + "type": "array", + "items": { + "$ref": "#/definitions/Path" + } + } + } + }, + "Filesystem": { + "properties": { + "name": { + "type": "string" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + } + } + }, + "FilesystemList": { + "properties": { + "filesystems": { + "type": "array", + "items": { + "$ref": "#/definitions/Filesystem" + } + } + } + } + }, + "responses": { + "ErrorResponse": { + "description": "An error occurred. The possible HTTP status, code, and message strings are listed below:\n* 400 Bad Request, ContentLengthMustBeZero, \"The Content-Length request header must be zero.\"\n* 400 Bad Request, InvalidAuthenticationInfo, \"Authentication information is not given in the correct format. Check the value of Authorization header.\"\n* 400 Bad Request, InvalidFlushPosition, \"The uploaded data is not contiguous or the position query parameter value is not equal to the length of the file after appending the uploaded data.\"\n* 400 Bad Request, InvalidHeaderValue, \"The value for one of the HTTP headers is not in the correct format.\"\n* 400 Bad Request, InvalidHttpVerb, \"The HTTP verb specified is invalid - it is not recognized by the server.\"\n* 400 Bad Request, InvalidInput, \"One of the request inputs is not valid.\"\n* 400 Bad Request, InvalidPropertyName, \"A property name cannot be empty.\"\n* 400 Bad Request, InvalidPropertyName, \"The property name contains invalid characters.\"\n* 400 Bad Request, InvalidQueryParameterValue, \"Value for one of the query parameters specified in the request URI is invalid.\"\n* 400 Bad Request, InvalidResourceName, \"The specified resource name contains invalid characters.\"\n* 400 Bad Request, InvalidSourceUri, \"The source URI is invalid.\"\n* 400 Bad Request, InvalidUri, \"The request URI is invalid.\"\n* 400 Bad Request, MissingRequiredHeader, \"An HTTP header that's mandatory for this request is not specified.\"\n* 400 Bad Request, MissingRequiredQueryParameter, \"A query parameter that's mandatory for this request is not specified.\"\n* 400 Bad Request, MultipleConditionHeadersNotSupported, \"Multiple condition headers are not supported.\"\n* 400 Bad Request, OutOfRangeInput, \"One of the request inputs is out of range.\"\n* 400 Bad Request, OutOfRangeQueryParameterValue, \"One of the query parameters specified in the request URI is outside the permissible range.\"\n* 400 Bad Request, UnsupportedHeader, \"One of the headers specified in the request is not supported.\"\n* 400 Bad Request, UnsupportedQueryParameter, \"One of the query parameters specified in the request URI is not supported.\"\n* 400 Bad Request, UnsupportedRestVersion, \"The specified Rest Version is Unsupported.\"\n* 403 Forbidden, AccountIsDisabled, \"The specified account is disabled.\"\n* 403 Forbidden, AuthorizationFailure, \"This request is not authorized to perform this operation.\"\n* 403 Forbidden, InsufficientAccountPermissions, \"The account being accessed does not have sufficient permissions to execute this operation.\"\n* 404 Not Found, FilesystemNotFound, \"The specified filesystem does not exist.\"\n* 404 Not Found, PathNotFound, \"The specified path does not exist.\"\n* 404 Not Found, RenameDestinationParentPathNotFound, \"The parent directory of the destination path does not exist.\"\n* 404 Not Found, ResourceNotFound, \"The specified resource does not exist.\"\n* 404 Not Found, SourcePathNotFound, \"The source path for a rename operation does not exist.\"\n* 405 Method Not Allowed, UnsupportedHttpVerb, \"The resource doesn't support the specified HTTP verb.\"\n* 409 Conflict, DestinationPathIsBeingDeleted, \"The specified destination path is marked to be deleted.\"\n* 409 Conflict, DirectoryNotEmpty, \"The recursive query parameter value must be true to delete a non-empty directory.\"\n* 409 Conflict, FilesystemAlreadyExists, \"The specified filesystem already exists.\"\n* 409 Conflict, FilesystemBeingDeleted, \"The specified filesystem is being deleted.\"\n* 409 Conflict, InvalidDestinationPath, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"* 409 Conflict, InvalidFlushOperation, \"The resource was created or modified by the Blob Service API and cannot be written to by the Data Lake Storage Service API.\"\n* 409 Conflict, InvalidRenameSourcePath, \"The source directory cannot be the same as the destination directory, nor can the destination be a subdirectory of the source directory.\"\n* 409 Conflict, InvalidSourceOrDestinationResourceType, \"The source and destination resource type must be identical.\"\n* 409 Conflict, LeaseAlreadyPresent, \"There is already a lease present.\"\n* 409 Conflict, LeaseIdMismatchWithLeaseOperation, \"The lease ID specified did not match the lease ID for the resource with the specified lease operation.\"\n* 409 Conflict, LeaseIsAlreadyBroken, \"The lease has already been broken and cannot be broken again.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeAcquired, \"The lease ID matched, but the lease is currently in breaking state and cannot be acquired until it is broken.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeChanged, \"The lease ID matched, but the lease is currently in breaking state and cannot be changed.\"\n* 409 Conflict, LeaseIsBrokenAndCannotBeRenewed, \"The lease ID matched, but the lease has been broken explicitly and cannot be renewed.\"\n* 409 Conflict, LeaseNameMismatch, \"The lease name specified did not match the existing lease name.\"\n* 409 Conflict, LeaseNotPresentWithLeaseOperation, \"The lease ID is not present with the specified lease operation.\"\n* 409 Conflict, PathAlreadyExists, \"The specified path already exists.\"\n* 409 Conflict, PathConflict, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"\n* 409 Conflict, SourcePathIsBeingDeleted, \"The specified source path is marked to be deleted.\"\n* 409 Conflict, ResourceTypeMismatch, \"The resource type specified in the request does not match the type of the resource.\"\n* 412 Precondition Failed, ConditionNotMet, \"The condition specified using HTTP conditional header(s) is not met.\"\n* 412 Precondition Failed, LeaseIdMismatch, \"The lease ID specified did not match the lease ID for the resource.\"\n* 412 Precondition Failed, LeaseIdMissing, \"There is currently a lease on the resource and no lease ID was specified in the request.\"\n* 412 Precondition Failed, LeaseNotPresent, \"There is currently no lease on the resource.\"\n* 412 Precondition Failed, LeaseLost, \"A lease ID was specified, but the lease for the resource has expired.\"\n* 412 Precondition Failed, SourceConditionNotMet, \"The source condition specified using HTTP conditional header(s) is not met.\"\n* 413 Request Entity Too Large, RequestBodyTooLarge, \"The request body is too large and exceeds the maximum permissible limit.\"\n* 416 Requested Range Not Satisfiable, InvalidRange, \"The range specified is invalid for the current size of the resource.\"\n* 500 Internal Server Error, InternalError, \"The server encountered an internal error. Please retry the request.\"\n* 500 Internal Server Error, OperationTimedOut, \"The operation could not be completed within the permitted time.\"\n* 503 Service Unavailable, ServerBusy, \"Egress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Ingress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Operations per second is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"The server is currently unable to receive requests. Please retry your request.\"", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/DataLakeStorageError" + } + } + }, + "paths": { + "/": { + "get": { + "operationId": "Filesystem_List", + "summary": "List Filesystems", + "description": "List filesystems and their properties in given account.", + "x-ms-pageable": { + "itemName": "filesystems", + "nextLinkName": null + }, + "tags": [ + "Account Operations" + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of filesystems to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "type": "string" + }, + "Content-Type": { + "description": "The content type of list filesystem response. The default content type is application/json.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/FilesystemList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "The value must be \"account\" for all account operations.", + "required": true, + "type": "string", + "enum": [ + "account" + ], + "x-ms-enum": { + "name": "AccountResourceType", + "modelAsString": false + } + }, + { + "name": "prefix", + "in": "query", + "description": "Filters results to filesystems within the specified prefix.", + "required": false, + "type": "string" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of filesystems returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + }, + "/{filesystem}": { + "put": { + "operationId": "Filesystem_Create", + "summary": "Create Filesystem", + "description": "Create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. This operation does not support conditional HTTP requests.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "201": { + "description": "Created", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Operations on files and directories do not affect the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Filesystem_SetProperties", + "summary": "Set Filesystem Properties", + "description": "Set properties for the filesystem. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. If the filesystem exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Path_List", + "summary": "List Paths", + "description": "List filesystem paths and their properties.", + "x-ms-pageable": { + "itemName": "paths", + "nextLinkName": null + }, + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of paths to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/PathList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "directory", + "in": "query", + "description": "Filters results to paths within the specified directory. An error occurs if the directory does not exist.", + "required": false, + "type": "string" + }, + { + "name": "recursive", + "in": "query", + "description": "If \"true\", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If \"directory\" is specified, the list will only include paths that share the same root.", + "required": true, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of paths returned with each invocation is limited. If the number of paths to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + } + ] + }, + "head": { + "operationId": "Filesystem_GetProperties", + "summary": "Get Filesystem Properties.", + "description": "All system and user-defined filesystem properties are specified in the response headers.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the filesystem. A comma-separated list of name and value pairs in the format \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + } + }, + "delete": { + "operationId": "Filesystem_Delete", + "summary": "Delete Filesystem", + "description": "Marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional error information indicating that the filesystem is being deleted. All other operations, including operations on any files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "202": { + "description": "Accepted", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have between 3 and 63 characters.", + "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "resource", + "in": "query", + "description": "The value must be \"filesystem\" for all filesystem operations.", + "required": true, + "type": "string", + "enum": [ + "filesystem" + ], + "x-ms-enum": { + "name": "FilesystemResourceType", + "modelAsString": false + } + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + }, + "/{filesystem}/{path}": { + "put": { + "operationId": "Path_Create", + "summary": "Create File | Create Directory | Rename File | Rename Directory", + "description": "Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: \"*\".", + "consumes": [ + "application/octet-stream" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "201": { + "description": "The file or directory was created.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "Required only for Create File and Create Directory. The value must be \"file\" or \"directory\".", + "required": false, + "type": "string", + "enum": [ + "directory", + "file" + ], + "x-ms-enum": { + "name": "PathResourceType", + "modelAsString": false + } + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "required": false, + "type": "string" + }, + { + "name": "mode", + "in": "query", + "description": "Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be \"legacy\" or \"posix\", and the default value will be \"posix\". ", + "required": false, + "type": "string", + "enum": [ + "legacy", + "posix" + ], + "x-ms-enum": { + "name": "PathRenameMode", + "modelAsString": false + } + }, + { + "name": "Cache-Control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "Content-Encoding", + "in": "header", + "description": "Optional. Specifies which content encodings have been applied to the file. This value is returned to the client when the \"Read File\" operation is performed.", + "required": false, + "type": "string" + }, + { + "name": "Content-Language", + "in": "header", + "description": "Optional. Specifies the natural language used by the intended audience for the file.", + "required": false, + "type": "string" + }, + { + "name": "Content-Disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-rename-source", + "in": "header", + "description": "An optional file or directory to be renamed. The value must have the following format: \"/{filesystem}/{path}\". If \"x-ms-properties\" is specified, the properties will overwrite the existing properties; otherwise, the existing properties will be preserved. This value must be a URL percent-encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. A lease ID for the path specified in the URI. The path to be overwritten must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-lease-id", + "in": "header", + "description": "Optional for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-umask", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. For example, if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027. The umask must be specified in 4-digit octal notation (e.g. 0766).", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-match", + "description": "Optional. An ETag value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-none-match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-modified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-unmodified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Path_Update", + "summary": "Append Data | Flush Data | Set Properties | Set Access Control", + "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "consumes": [ + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The data was flushed (written) to the file or the properties were set successfully.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "x-ms-properties": { + "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "202": { + "description": "The uploaded data was accepted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "The action must be \"append\" to upload data to be appended to a file, \"flush\" to flush previously uploaded data to a file, \"setProperties\" to set the properties of a file or directory, or \"setAccessControl\" to set the owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are mutually exclusive.", + "required": true, + "type": "string", + "enum": [ + "append", + "flush", + "setProperties", + "setAccessControl" + ], + "x-ms-enum": { + "name": "PathUpdateAction", + "modelAsString": false + } + }, + { + "name": "position", + "in": "query", + "description": "This parameter allows the caller to upload data in parallel and control the order in which it is appended to the file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to the file. The value must be the position where the data is to be appended. Uploaded data is not immediately flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter must be specified and equal to the length of the file after all data has been written, and there must not be a request entity body included with the request.", + "format": "int64", + "required": false, + "type": "integer" + }, + { + "name": "retainUncommittedData", + "in": "query", + "description": "Valid only for flush operations. If \"true\", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after the flush operation. The default is false. Data at offsets less than the specified position are written to the file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future flush operation.", + "required": false, + "type": "boolean" + }, + { + "name": "close", + "in": "query", + "description": "Azure Storage Events allow applications to receive notifications when files change. When Azure Storage Events are enabled, a file changed event is raised. This event has a property indicating whether this is the final change to distinguish the difference between an intermediate flush to a file stream and the final close of a file stream. The close query parameter is valid only when the action is \"flush\" and change notifications are enabled. If the value of close is \"true\" and the flush operation completes successfully, the service raises a file change notification with a property indicating that this is the final update (the file stream has been closed). If \"false\" a change notification is raised indicating the file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS driver to indicate that the file stream has been closed.\"", + "required": false, + "type": "boolean" + }, + { + "name": "Content-Length", + "in": "header", + "description": "Required for \"Append Data\" and \"Flush Data\". Must be 0 for \"Flush Data\". Must be the length of the request content in bytes for \"Append Data\".", + "minimum": 0, + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-md5", + "in": "header", + "description": "Optional and only valid for \"Flush & Set Properties\" operations. The service stores this value and includes it in the \"Content-Md5\" response header for \"Read & Get Properties\" operations. If this property is not specified on the request, then the property will be cleared for the file. Subsequent calls to \"Read & Get Properties\" will not return this property unless it is explicitly set on that file again.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. Valid only for the setProperties operation. If the file or directory exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-owner", + "description": "Optional and valid only for the setAccessControl operation. Sets the owner of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-group", + "description": "Optional and valid only for the setAccessControl operation. Sets the owning group of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction with x-ms-acl.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-acl", + "description": "Optional and valid only for the setAccessControl operation. Sets POSIX access control rights on files and directories. The value is a comma-separated list of access control entries that fully replaces the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or group identifier, and permissions in the format \"[scope:][type]:[id]:[permissions]\". The scope must be \"default\" to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL. There are four ACE types: \"user\" grants rights to the owner or a named user, \"group\" grants rights to the owning group or a named group, \"mask\" restricts rights granted to named users and the members of groups, and \"other\" grants rights to all users not found in any of the other entries. The user or group identifier is omitted for entries of type \"mask\" and \"other\". The user or group identifier is also omitted for the owner and owning group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If access is not granted, the '-' character is used to denote that the permission is denied. For example, the following ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning group, and nothing to everyone else: \"user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx\". Invalid in conjunction with x-ms-permissions.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "requestBody", + "description": "Valid only for append operations. The data to be uploaded and appended to the file.", + "in": "body", + "required": false, + "schema": { + "type": "object", + "format": "file" + } + } + ] + }, + "post": { + "operationId": "Path_Lease", + "summary": "Lease Path", + "description": "Create and manage a lease to restrict write and delete access to the path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The \"renew\", \"change\" or \"release\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file was last modified. Write operations on the file update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"renew\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "201": { + "description": "A new lease has been created. The \"acquire\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"acquire\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "202": { + "description": "The \"break\" lease action was successful.", + "headers": { + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-time": { + "description": "The time remaining in the lease period in seconds.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-lease-action", + "in": "header", + "description": "There are five lease actions: \"acquire\", \"break\", \"change\", \"renew\", and \"release\". Use \"acquire\" and specify the \"x-ms-proposed-lease-id\" and \"x-ms-lease-duration\" to acquire a new lease. Use \"break\" to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease operation except break and release can be performed on the file. When a lease is successfully broken, the response indicates the interval in seconds until a new lease can be acquired. Use \"change\" and specify the current lease ID in \"x-ms-lease-id\" and the new lease ID in \"x-ms-proposed-lease-id\" to change the lease ID of an active lease. Use \"renew\" and specify the \"x-ms-lease-id\" to renew an existing lease. Use \"release\" and specify the \"x-ms-lease-id\" to release a lease.", + "required": true, + "type": "string", + "enum": [ + "acquire", + "break", + "change", + "renew", + "release" + ], + "x-ms-enum": { + "name": "PathLeaseAction", + "modelAsString": false + } + }, + { + "name": "x-ms-lease-duration", + "in": "header", + "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-break-period", + "in": "header", + "description": "The lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. The lease break duration must be between 0 and 60 seconds.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"renew\", \"change\" or \"release\". For the renew and release actions, this must match the current lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-proposed-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"acquire\" or \"change\". A lease will be acquired with this lease ID if the operation is successful.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Path_Read", + "summary": "Read File", + "description": "Read the contents of a file. For read operations, range requests are supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "produces": [ + "application/json", + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file. If the file has an MD5 hash and this read operation is to read the complete file, this response header is returned so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "206": { + "description": "Partial content", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "in": "header", + "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved.", + "required": false, + "type": "string", + "name": "Range" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "head": { + "operationId": "Path_GetProperties", + "summary": "Get Properties | Get Status | Get Access Control List", + "description": "Get Properties returns all system and user defined properties for a path. Get Status returns all system defined properties for a path. Get Access Control List returns the access control list for a path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Returns all properties for the file or directory.", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file stored in storage. This header is returned only for \"GetProperties\" operation. If the Content-MD5 header has been set for the file, this response header is returned for GetProperties call so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-owner": { + "description": "The owner of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-group": { + "description": "The owning group of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-permissions": { + "description": "The POSIX access permissions for the file owner, the file owning group, and others. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-acl": { + "description": "The POSIX access control list for the file or directory. Included in the response only if the action is \"getAccessControl\" and Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "Optional. If the value is \"getStatus\" only the system defined properties for the path are returned. If the value is \"getAccessControl\" the access control list is returned in the response headers (Hierarchical Namespace must be enabled for the account), otherwise the properties are returned.", + "required": false, + "type": "string", + "enum": [ + "getAccessControl", + "getStatus" + ], + "x-ms-enum": { + "name": "PathGetPropertiesAction", + "modelAsString": false + } + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the x-ms-owner, x-ms-group, and x-ms-acl response headers will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "delete": { + "operationId": "Path_Delete", + "summary": "Delete File | Delete Directory", + "description": "Delete the file or directory. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The file was deleted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "recursive", + "in": "query", + "description": "Required and valid only when the resource is a directory. If \"true\", all paths beneath the directory will be deleted. If \"false\" and the directory is non-empty, an error occurs.", + "required": false, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier.", + "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "path", + "in": "path", + "description": "The file or directory path.", + "required": true, + "type": "string" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + } +} \ No newline at end of file diff --git a/azbfs/zz_generated_client.go b/azbfs/zz_generated_client.go index 3cfcb281c..1288425f2 100644 --- a/azbfs/zz_generated_client.go +++ b/azbfs/zz_generated_client.go @@ -4,20 +4,13 @@ package azbfs // Changes may cause incorrect behavior and will be lost if the code is regenerated. import ( - "context" - "encoding/json" "github.com/Azure/azure-pipeline-go/pipeline" - "io" - "io/ioutil" - "net/http" "net/url" - "strconv" ) const ( // ServiceVersion specifies the version of the operations used in this package. - ServiceVersion = "2018-03-28" - //ServiceVersion = "2018-06-17" //TODO uncomment when service is ready + ServiceVersion = "2018-11-09" ) // managementClient is the base client for Azbfs. @@ -43,1270 +36,3 @@ func (mc managementClient) URL() url.URL { func (mc managementClient) Pipeline() pipeline.Pipeline { return mc.p } - -// CreateFilesystem create a filesystem rooted at the specified location. If the filesystem already exists, the -// operation fails. This operation does not support conditional HTTP requests. -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. xMsProperties is user-defined properties to be stored with the filesystem, in the format of a -// comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is base64 encoded. -// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) CreateFilesystem(ctx context.Context, filesystem string, resource string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*CreateFilesystemResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.createFilesystemPreparer(filesystem, resource, xMsProperties, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createFilesystemResponder}, req) - if err != nil { - return nil, err - } - return resp.(*CreateFilesystemResponse), err -} - -// createFilesystemPreparer prepares the CreateFilesystem request. -func (client managementClient) createFilesystemPreparer(filesystem string, resource string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PUT", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// createFilesystemResponder handles the response to the CreateFilesystem request. -func (client managementClient) createFilesystemResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusCreated) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &CreateFilesystemResponse{rawResponse: resp.Response()}, err -} - -// CreatePath create or rename a file or directory. By default, the destination is overwritten and if the -// destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. -// For more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// To fail if the destination already exists, use a conditional request with If-None-Match: "*". -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. resource is required only for -// Create File and Create Directory. The value must be "file" or "directory". continuation is optional. When renaming -// a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be -// renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is -// returned in the response, it must be specified in a subsequent invocation of the rename operation to continue -// renaming the directory. mode is optional. Valid only when namespace is enabled. This parameter determines the -// behavior of the rename operation. The value must be "legacy" or "posix", and the default value will be "posix". -// cacheControl is optional. The service stores this value and includes it in the "Cache-Control" response header for -// "Read File" operations for "Read File" operations. contentEncoding is optional. Specifies which content encodings -// have been applied to the file. This value is returned to the client when the "Read File" operation is performed. -// contentLanguage is optional. Specifies the natural language used by the intended audience for the file. -// contentDisposition is optional. The service stores this value and includes it in the "Content-Disposition" response -// header for "Read File" operations. xMsCacheControl is optional. The service stores this value and includes it in -// the "Cache-Control" response header for "Read File" operations. xMsContentType is optional. The service stores this -// value and includes it in the "Content-Type" response header for "Read File" operations. xMsContentEncoding is -// optional. The service stores this value and includes it in the "Content-Encoding" response header for "Read File" -// operations. xMsContentLanguage is optional. The service stores this value and includes it in the "Content-Language" -// response header for "Read File" operations. xMsContentDisposition is optional. The service stores this value and -// includes it in the "Content-Disposition" response header for "Read File" operations. xMsRenameSource is an optional -// file or directory to be renamed. The value must have the following format: "/{filesysystem}/{path}". If -// "x-ms-properties" is specified, the properties will overwrite the existing properties; otherwise, the existing -// properties will be preserved. xMsLeaseID is optional. A lease ID for the path specified in the URI. The path to be -// overwritten must have an active lease and the lease ID must match. xMsProposedLeaseID is optional for create -// operations. Required when "x-ms-lease-action" is used. A lease will be acquired using the proposed ID when the -// resource is created. xMsSourceLeaseID is optional for rename operations. A lease ID for the source path. The -// source path must have an active lease and the lease ID must match. xMsProperties is optional. User-defined -// properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs -// "n1=v1, n2=v2, ...", where each value is base64 encoded. xMsPermissions is optional and only valid if Hierarchical -// Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and -// others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both -// symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. ifMatch is optional. An ETag value. -// Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must -// be specified in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this -// header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be -// specified in quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform the -// operation only if the resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A -// date and time value. Specify this header to perform the operation only if the resource has not been modified since -// the specified date and time. xMsSourceIfMatch is optional. An ETag value. Specify this header to perform the rename -// operation only if the source's ETag matches the value specified. The ETag must be specified in quotes. -// xMsSourceIfNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to perform -// the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in -// quotes. xMsSourceIfModifiedSince is optional. A date and time value. Specify this header to perform the rename -// operation only if the source has been modified since the specified date and time. xMsSourceIfUnmodifiedSince is -// optional. A date and time value. Specify this header to perform the rename operation only if the source has not been -// modified since the specified date and time. xMsClientRequestID is a UUID recorded in the analytics logs for -// troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The period begins when -// the request is received by the service. If the timeout value elapses before the operation completes, the operation -// fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is required when using -// shared key authorization. -func (client managementClient) CreatePath(ctx context.Context, filesystem string, pathParameter string, resource *string, continuation *string, mode *string, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsProposedLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*CreatePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: xMsProposedLeaseID, - constraints: []constraint{{target: "xMsProposedLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: xMsSourceLeaseID, - constraints: []constraint{{target: "xMsSourceLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsSourceLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.createPathPreparer(filesystem, pathParameter, resource, continuation, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, xMsCacheControl, xMsContentType, xMsContentEncoding, xMsContentLanguage, xMsContentDisposition, xMsRenameSource, xMsLeaseID, xMsProposedLeaseID, xMsSourceLeaseID, xMsProperties, xMsPermissions, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsSourceIfMatch, xMsSourceIfNoneMatch, xMsSourceIfModifiedSince, xMsSourceIfUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createPathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*CreatePathResponse), err -} - -// createPathPreparer prepares the CreatePath request. -func (client managementClient) createPathPreparer(filesystem string, pathParameter string, resource *string, continuation *string, mode *string, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsProposedLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PUT", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if resource != nil && len(*resource) > 0 { - params.Set("resource", *resource) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if mode != nil && len(*mode) > 0 { - params.Set("mode", *mode) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if cacheControl != nil { - req.Header.Set("Cache-Control", *cacheControl) - } - if contentEncoding != nil { - req.Header.Set("Content-Encoding", *contentEncoding) - } - if contentLanguage != nil { - req.Header.Set("Content-Language", *contentLanguage) - } - if contentDisposition != nil { - req.Header.Set("Content-Disposition", *contentDisposition) - } - if xMsCacheControl != nil { - req.Header.Set("x-ms-cache-control", *xMsCacheControl) - } - if xMsContentType != nil { - req.Header.Set("x-ms-content-type", *xMsContentType) - } - if xMsContentEncoding != nil { - req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) - } - if xMsContentLanguage != nil { - req.Header.Set("x-ms-content-language", *xMsContentLanguage) - } - if xMsContentDisposition != nil { - req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) - } - if xMsRenameSource != nil { - req.Header.Set("x-ms-rename-source", *xMsRenameSource) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsProposedLeaseID != nil { - req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) - } - if xMsSourceLeaseID != nil { - req.Header.Set("x-ms-source-lease-id", *xMsSourceLeaseID) - } - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsPermissions != nil { - req.Header.Set("x-ms-permissions", *xMsPermissions) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsSourceIfMatch != nil { - req.Header.Set("x-ms-source-if-match", *xMsSourceIfMatch) - } - if xMsSourceIfNoneMatch != nil { - req.Header.Set("x-ms-source-if-none-match", *xMsSourceIfNoneMatch) - } - if xMsSourceIfModifiedSince != nil { - req.Header.Set("x-ms-source-if-modified-since", *xMsSourceIfModifiedSince) - } - if xMsSourceIfUnmodifiedSince != nil { - req.Header.Set("x-ms-source-if-unmodified-since", *xMsSourceIfUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// createPathResponder handles the response to the CreatePath request. -func (client managementClient) createPathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusCreated) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &CreatePathResponse{rawResponse: resp.Response()}, err -} - -// DeleteFilesystem marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same -// identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a -// filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional -// error information indicating that the filesystem is being deleted. All other operations, including operations on any -// files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being -// deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional -// Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if -// the resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time -// value. Specify this header to perform the operation only if the resource has not been modified since the specified -// date and time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. -// timeout is an optional operation timeout value in seconds. The period begins when the request is received by the -// service. If the timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the -// Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) DeleteFilesystem(ctx context.Context, filesystem string, resource string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*DeleteFilesystemResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.deleteFilesystemPreparer(filesystem, resource, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteFilesystemResponder}, req) - if err != nil { - return nil, err - } - return resp.(*DeleteFilesystemResponse), err -} - -// deleteFilesystemPreparer prepares the DeleteFilesystem request. -func (client managementClient) deleteFilesystemPreparer(filesystem string, resource string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("DELETE", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// deleteFilesystemResponder handles the response to the DeleteFilesystem request. -func (client managementClient) deleteFilesystemResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusAccepted) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &DeleteFilesystemResponse{rawResponse: resp.Response()}, err -} - -// DeletePath delete the file or directory. This operation supports conditional HTTP requests. For more information, -// see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. recursive is required and -// valid only when the resource is a directory. If "true", all paths beneath the directory will be deleted. If "false" -// and the directory is non-empty, an error occurs. continuation is optional. When deleting a directory, the number of -// paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a -// continuation token is returned in this response header. When a continuation token is returned in the response, it -// must be specified in a subsequent invocation of the delete operation to continue deleting the directory. xMsLeaseID -// is the lease ID must be specified if there is an active lease. ifMatch is optional. An ETag value. Specify this -// header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified -// in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to -// perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in -// quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the -// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. -// Specify this header to perform the operation only if the resource has not been modified since the specified date and -// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) DeletePath(ctx context.Context, filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*DeletePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.deletePathPreparer(filesystem, pathParameter, recursive, continuation, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deletePathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*DeletePathResponse), err -} - -// deletePathPreparer prepares the DeletePath request. -func (client managementClient) deletePathPreparer(filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("DELETE", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if recursive != nil { - params.Set("recursive", strconv.FormatBool(*recursive)) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// deletePathResponder handles the response to the DeletePath request. -func (client managementClient) deletePathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &DeletePathResponse{rawResponse: resp.Response()}, err -} - -// GetFilesystemProperties all system and user-defined filesystem properties are specified in the response headers. -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout -// is an optional operation timeout value in seconds. The period begins when the request is received by the service. If -// the timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) GetFilesystemProperties(ctx context.Context, filesystem string, resource string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*GetFilesystemPropertiesResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.getFilesystemPropertiesPreparer(filesystem, resource, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getFilesystemPropertiesResponder}, req) - if err != nil { - return nil, err - } - return resp.(*GetFilesystemPropertiesResponse), err -} - -// getFilesystemPropertiesPreparer prepares the GetFilesystemProperties request. -func (client managementClient) getFilesystemPropertiesPreparer(filesystem string, resource string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("HEAD", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// getFilesystemPropertiesResponder handles the response to the GetFilesystemProperties request. -func (client managementClient) getFilesystemPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &GetFilesystemPropertiesResponse{rawResponse: resp.Response()}, err -} - -// GetPathProperties get the properties for a file or directory, and optionally include the access control list. This -// operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob -// Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. action is optional. If the -// value is "getAccessControl" the access control list is returned in the response headers (Hierarchical Namespace must -// be enabled for the account). ifMatch is optional. An ETag value. Specify this header to perform the operation only -// if the resource's ETag matches the value specified. The ETag must be specified in quotes. ifNoneMatch is optional. -// An ETag value or the special wildcard ("*") value. Specify this header to perform the operation only if the -// resource's ETag does not match the value specified. The ETag must be specified in quotes. ifModifiedSince is -// optional. A date and time value. Specify this header to perform the operation only if the resource has been modified -// since the specified date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to -// perform the operation only if the resource has not been modified since the specified date and time. -// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) GetPathProperties(ctx context.Context, filesystem string, pathParameter string, action *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*GetPathPropertiesResponse, error) { - if err := validate([]validation{ - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.getPathPropertiesPreparer(filesystem, pathParameter, action, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPathPropertiesResponder}, req) - if err != nil { - return nil, err - } - return resp.(*GetPathPropertiesResponse), err -} - -// getPathPropertiesPreparer prepares the GetPathProperties request. -func (client managementClient) getPathPropertiesPreparer(filesystem string, pathParameter string, action *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("HEAD", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if action != nil && len(*action) > 0 { - params.Set("action", *action) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// getPathPropertiesResponder handles the response to the GetPathProperties request. -func (client managementClient) getPathPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &GetPathPropertiesResponse{rawResponse: resp.Response()}, err -} - -// LeasePath create and manage a lease to restrict write and delete access to the path. This operation supports -// conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// xMsLeaseAction is there are five lease actions: "acquire", "break", "change", "renew", and "release". Use "acquire" -// and specify the "x-ms-proposed-lease-id" and "x-ms-lease-duration" to acquire a new lease. Use "break" to break an -// existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease -// operation except break and release can be performed on the file. When a lease is successfully broken, the response -// indicates the interval in seconds until a new lease can be acquired. Use "change" and specify the current lease ID -// in "x-ms-lease-id" and the new lease ID in "x-ms-proposed-lease-id" to change the lease ID of an active lease. Use -// "renew" and specify the "x-ms-lease-id" to renew an existing lease. Use "release" and specify the "x-ms-lease-id" to -// release a lease. filesystem is the filesystem identifier. pathParameter is the file or directory path. -// xMsLeaseDuration is the lease duration is required to acquire a lease, and specifies the duration of the lease in -// seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. xMsLeaseBreakPeriod is the -// lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. -// The lease break duration must be between 0 and 60 seconds. xMsLeaseID is required when "x-ms-lease-action" is -// "renew", "change" or "release". For the renew and release actions, this must match the current lease ID. -// xMsProposedLeaseID is required when "x-ms-lease-action" is "acquire" or "change". A lease will be acquired with -// this lease ID if the operation is successful. ifMatch is optional. An ETag value. Specify this header to perform -// the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes. -// ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to perform the -// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. -// ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the -// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. -// Specify this header to perform the operation only if the resource has not been modified since the specified date and -// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) LeasePath(ctx context.Context, xMsLeaseAction string, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*LeasePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: xMsProposedLeaseID, - constraints: []constraint{{target: "xMsProposedLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.leasePathPreparer(xMsLeaseAction, filesystem, pathParameter, xMsLeaseDuration, xMsLeaseBreakPeriod, xMsLeaseID, xMsProposedLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.leasePathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*LeasePathResponse), err -} - -// leasePathPreparer prepares the LeasePath request. -func (client managementClient) leasePathPreparer(xMsLeaseAction string, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("POST", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - req.Header.Set("x-ms-lease-action", xMsLeaseAction) - if xMsLeaseDuration != nil { - req.Header.Set("x-ms-lease-duration", strconv.FormatInt(int64(*xMsLeaseDuration), 10)) - } - if xMsLeaseBreakPeriod != nil { - req.Header.Set("x-ms-lease-break-period", strconv.FormatInt(int64(*xMsLeaseBreakPeriod), 10)) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsProposedLeaseID != nil { - req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// leasePathResponder handles the response to the LeasePath request. -func (client managementClient) leasePathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &LeasePathResponse{rawResponse: resp.Response()}, err -} - -// ListFilesystems list filesystems and their properties in given account. -// -// resource is the value must be "account" for all account operations. prefix is filters results to filesystems within -// the specified prefix. continuation is the number of filesystems returned with each invocation is limited. If the -// number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header -// x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent -// invocation of the list operation to continue listing the filesystems. maxResults is an optional value that specifies -// the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 -// items. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is -// an optional operation timeout value in seconds. The period begins when the request is received by the service. If -// the timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) ListFilesystems(ctx context.Context, resource string, prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ListFilesystemSchema, error) { - if err := validate([]validation{ - {targetValue: maxResults, - constraints: []constraint{{target: "maxResults", name: null, rule: false, - chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.listFilesystemsPreparer(resource, prefix, continuation, maxResults, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listFilesystemsResponder}, req) - if err != nil { - return nil, err - } - return resp.(*ListFilesystemSchema), err -} - -// listFilesystemsPreparer prepares the ListFilesystems request. -func (client managementClient) listFilesystemsPreparer(resource string, prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("GET", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if prefix != nil && len(*prefix) > 0 { - params.Set("prefix", *prefix) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if maxResults != nil { - params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// listFilesystemsResponder handles the response to the ListFilesystems request. -func (client managementClient) listFilesystemsResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - result := &ListFilesystemSchema{rawResponse: resp.Response()} - if err != nil { - return result, err - } - defer resp.Response().Body.Close() - b, err := ioutil.ReadAll(resp.Response().Body) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to read response body") - } - if len(b) > 0 { - b = removeBOM(b) - err = json.Unmarshal(b, result) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") - } - } - return result, nil -} - -// ListPaths list filesystem paths and their properties. -// -// recursive is if "true", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If -// "directory" is specified, the list will only include paths that share the same root. filesystem is the filesystem -// identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the -// dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have -// between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem operations. directory is -// filters results to paths within the specified directory. An error occurs if the directory does not exist. -// continuation is the number of paths returned with each invocation is limited. If the number of paths to be returned -// exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation -// token is returned in the response, it must be specified in a subsequent invocation of the list operation to -// continue listing the paths. maxResults is an optional value that specifies the maximum number of items to return. If -// omitted or greater than 5,000, the response will include up to 5,000 items. xMsClientRequestID is a UUID recorded in -// the analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. -// The period begins when the request is received by the service. If the timeout value elapses before the operation -// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is -// required when using shared key authorization. -func (client managementClient) ListPaths(ctx context.Context, recursive bool, filesystem string, resource string, directory *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ListSchema, error) { - if err := validate([]validation{ - {targetValue: maxResults, - constraints: []constraint{{target: "maxResults", name: null, rule: false, - chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.listPathsPreparer(recursive, filesystem, resource, directory, continuation, maxResults, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listPathsResponder}, req) - if err != nil { - return nil, err - } - return resp.(*ListSchema), err -} - -// listPathsPreparer prepares the ListPaths request. -func (client managementClient) listPathsPreparer(recursive bool, filesystem string, resource string, directory *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("GET", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if directory != nil && len(*directory) > 0 { - params.Set("directory", *directory) - } - params.Set("recursive", strconv.FormatBool(recursive)) - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if maxResults != nil { - params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) - } - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// listPathsResponder handles the response to the ListPaths request. -func (client managementClient) listPathsResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - result := &ListSchema{rawResponse: resp.Response()} - if err != nil { - return result, err - } - defer resp.Response().Body.Close() - b, err := ioutil.ReadAll(resp.Response().Body) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to read response body") - } - if len(b) > 0 { - b = removeBOM(b) - err = json.Unmarshal(b, result) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") - } - } - return result, nil -} - -// ReadPath read the contents of a file. For read operations, range requests are supported. This operation supports -// conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. pathParameter is the file or directory path. rangeParameter is the HTTP -// Range request header specifies one or more byte ranges of the resource to be retrieved. ifMatch is optional. An -// ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. -// The ETag must be specified in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. -// Specify this header to perform the operation only if the resource's ETag does not match the value specified. The -// ETag must be specified in quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform -// the operation only if the resource has been modified since the specified date and time. ifUnmodifiedSince is -// optional. A date and time value. Specify this header to perform the operation only if the resource has not been -// modified since the specified date and time. xMsClientRequestID is a UUID recorded in the analytics logs for -// troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The period begins when -// the request is received by the service. If the timeout value elapses before the operation completes, the operation -// fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is required when using -// shared key authorization. -func (client managementClient) ReadPath(ctx context.Context, filesystem string, pathParameter string, rangeParameter *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ReadPathResponse, error) { - if err := validate([]validation{ - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.readPathPreparer(filesystem, pathParameter, rangeParameter, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.readPathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*ReadPathResponse), err -} - -// readPathPreparer prepares the ReadPath request. -func (client managementClient) readPathPreparer(filesystem string, pathParameter string, rangeParameter *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("GET", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if rangeParameter != nil { - req.Header.Set("Range", *rangeParameter) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// readPathResponder handles the response to the ReadPath request. -func (client managementClient) readPathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusPartialContent) - if resp == nil { - return nil, err - } - return &ReadPathResponse{rawResponse: resp.Response()}, err -} - -// SetFilesystemProperties set properties for the filesystem. This operation supports conditional HTTP requests. For -// more information, see [Specifying Conditional Headers for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only -// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. -// The value must have between 3 and 63 characters. resource is the value must be "filesystem" for all filesystem -// operations. xMsProperties is optional. User-defined properties to be stored with the filesystem, in the format of a -// comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is base64 encoded. If the -// filesystem exists, any properties not included in the list will be removed. All properties are removed if the -// header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, -// then make a conditional request with the E-Tag and include values for all properties. ifModifiedSince is optional. A -// date and time value. Specify this header to perform the operation only if the resource has been modified since the -// specified date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the -// operation only if the resource has not been modified since the specified date and time. xMsClientRequestID is a UUID -// recorded in the analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value -// in seconds. The period begins when the request is received by the service. If the timeout value elapses before the -// operation completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. -// This is required when using shared key authorization. -func (client managementClient) SetFilesystemProperties(ctx context.Context, filesystem string, resource string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*SetFilesystemPropertiesResponse, error) { - if err := validate([]validation{ - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.setFilesystemPropertiesPreparer(filesystem, resource, xMsProperties, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.setFilesystemPropertiesResponder}, req) - if err != nil { - return nil, err - } - return resp.(*SetFilesystemPropertiesResponse), err -} - -// setFilesystemPropertiesPreparer prepares the SetFilesystemProperties request. -func (client managementClient) setFilesystemPropertiesPreparer(filesystem string, resource string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PATCH", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("resource", resource) - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// setFilesystemPropertiesResponder handles the response to the SetFilesystemProperties request. -func (client managementClient) setFilesystemPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &SetFilesystemPropertiesResponse{rawResponse: resp.Response()}, err -} - -// UpdatePath uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets -// properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a -// file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers -// for Blob Service -// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). -// -// action is the action must be "append" to upload data to be appended to a file, "flush" to flush previously uploaded -// data to a file, "setProperties" to set the properties of a file or directory, or "setAccessControl" to set the -// owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be -// enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes -// permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are -// mutually exclusive. filesystem is the filesystem identifier. pathParameter is the file or directory path. position -// is this parameter allows the caller to upload data in parallel and control the order in which it is appended to the -// file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to -// the file. The value must be the position where the data is to be appended. Uploaded data is not immediately -// flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter -// must be specified and equal to the length of the file after all data has been written, and there must not be a -// request entity body included with the request. retainUncommittedData is valid only for flush operations. If "true", -// uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after -// the flush operation. The default is false. Data at offsets less than the specified position are written to the -// file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a -// future flush operation. contentLength is required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". -// Must be the length of the request content in bytes for "Append Data". xMsLeaseAction is optional. The lease action -// can be "renew" to renew an existing lease or "release" to release a lease. xMsLeaseID is the lease ID must be -// specified if there is an active lease. xMsCacheControl is optional and only valid for flush and set properties -// operations. The service stores this value and includes it in the "Cache-Control" response header for "Read File" -// operations. xMsContentType is optional and only valid for flush and set properties operations. The service stores -// this value and includes it in the "Content-Type" response header for "Read File" operations. xMsContentDisposition -// is optional and only valid for flush and set properties operations. The service stores this value and includes it -// in the "Content-Disposition" response header for "Read File" operations. xMsContentEncoding is optional and only -// valid for flush and set properties operations. The service stores this value and includes it in the -// "Content-Encoding" response header for "Read File" operations. xMsContentLanguage is optional and only valid for -// flush and set properties operations. The service stores this value and includes it in the "Content-Language" -// response header for "Read File" operations. xMsProperties is optional. User-defined properties to be stored with -// the file or directory, in the format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where -// each value is base64 encoded. Valid only for the setProperties operation. If the file or directory exists, any -// properties not included in the list will be removed. All properties are removed if the header is omitted. To merge -// new and existing properties, first get all existing properties and the current E-Tag, then make a conditional -// request with the E-Tag and include values for all properties. xMsOwner is optional and valid only for the -// setAccessControl operation. Sets the owner of the file or directory. xMsGroup is optional and valid only for the -// setAccessControl operation. Sets the owning group of the file or directory. xMsPermissions is optional and only -// valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the -// file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also -// supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction -// with x-ms-acl. xMsACL is optional and valid only for the setAccessControl operation. Sets POSIX access control -// rights on files and directories. The value is a comma-separated list of access control entries that fully replaces -// the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or -// group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". The scope must be "default" to -// indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the -// access ACL. There are four ACE types: "user" grants rights to the owner or a named user, "group" grants rights to -// the owning group or a named group, "mask" restricts rights granted to named users and the members of groups, and -// "other" grants rights to all users not found in any of the other entries. The user or group identifier is omitted -// for entries of type "mask" and "other". The user or group identifier is also omitted for the owner and owning -// group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the -// second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If -// access is not granted, the '-' character is used to denote that the permission is denied. For example, the following -// ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning -// group, and nothing to everyone else: "user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx". Invalid -// in conjunction with x-ms-permissions. ifMatch is optional for Flush Data and Set Properties, but invalid for Append -// Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value -// specified. The ETag must be specified in quotes. ifNoneMatch is optional for Flush Data and Set Properties, but -// invalid for Append Data. An ETag value or the special wildcard ("*") value. Specify this header to perform the -// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. -// ifModifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. -// Specify this header to perform the operation only if the resource has been modified since the specified date and -// time. ifUnmodifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time -// value. Specify this header to perform the operation only if the resource has not been modified since the specified -// date and time. xHTTPMethodOverride is optional. Override the http verb on the service side. Some older http clients -// do not support PATCH requestBody is valid only for append operations. The data to be uploaded and appended to the -// file. requestBody will be closed upon successful return. Callers should ensure closure when receiving an -// error.xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an -// optional operation timeout value in seconds. The period begins when the request is received by the service. If the -// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated -// Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client managementClient) UpdatePath(ctx context.Context, action string, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, contentLength *string, xMsLeaseAction *string, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*UpdatePathResponse, error) { - if err := validate([]validation{ - {targetValue: xMsLeaseID, - constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, - chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.updatePathPreparer(action, filesystem, pathParameter, position, retainUncommittedData, contentLength, xMsLeaseAction, xMsLeaseID, xMsCacheControl, xMsContentType, xMsContentDisposition, xMsContentEncoding, xMsContentLanguage, xMsProperties, xMsOwner, xMsGroup, xMsPermissions, xMsACL, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xHTTPMethodOverride, body, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.updatePathResponder}, req) - if err != nil { - return nil, err - } - return resp.(*UpdatePathResponse), err -} - -// updatePathPreparer prepares the UpdatePath request. -func (client managementClient) updatePathPreparer(action string, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, contentLength *string, xMsLeaseAction *string, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PUT", client.url, body) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - params.Set("action", action) - if position != nil { - params.Set("position", strconv.FormatInt(*position, 10)) - } - if retainUncommittedData != nil { - params.Set("retainUncommittedData", strconv.FormatBool(*retainUncommittedData)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if contentLength != nil { - req.Header.Set("Content-Length", *contentLength) - } - if xMsLeaseAction != nil { - req.Header.Set("x-ms-lease-action", *xMsLeaseAction) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsCacheControl != nil { - req.Header.Set("x-ms-cache-control", *xMsCacheControl) - } - if xMsContentType != nil { - req.Header.Set("x-ms-content-type", *xMsContentType) - } - if xMsContentDisposition != nil { - req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) - } - if xMsContentEncoding != nil { - req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) - } - if xMsContentLanguage != nil { - req.Header.Set("x-ms-content-language", *xMsContentLanguage) - } - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsOwner != nil { - req.Header.Set("x-ms-owner", *xMsOwner) - } - if xMsGroup != nil { - req.Header.Set("x-ms-group", *xMsGroup) - } - if xMsPermissions != nil { - req.Header.Set("x-ms-permissions", *xMsPermissions) - } - if xMsACL != nil { - req.Header.Set("x-ms-acl", *xMsACL) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xHTTPMethodOverride != nil { - req.Header.Set("x-http-method-override", *xHTTPMethodOverride) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// updatePathResponder handles the response to the UpdatePath request. -func (client managementClient) updatePathResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK, http.StatusAccepted) - if resp == nil { - return nil, err - } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &UpdatePathResponse{rawResponse: resp.Response()}, err -} diff --git a/azbfs/zz_generated_filesystem.go b/azbfs/zz_generated_filesystem.go new file mode 100644 index 000000000..a52000077 --- /dev/null +++ b/azbfs/zz_generated_filesystem.go @@ -0,0 +1,453 @@ +package azbfs + +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "net/url" + "github.com/Azure/azure-pipeline-go/pipeline" + "net/url" + "net/http" + "net/url" + "context" + "net/url" + "strconv" + "net/url" + "encoding/json" + "net/url" + "io/ioutil" + "net/url" + "io" +) + +// filesystemClient is the azure Data Lake Storage provides storage for Hadoop and other big data workloads. +type filesystemClient struct { + managementClient +} +// newFilesystemClient creates an instance of the filesystemClient client. +func newFilesystemClient(url url.URL, p pipeline.Pipeline) filesystemClient { + return filesystemClient{newManagementClient(url, p)} +} + +// Create create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. +// This operation does not support conditional HTTP requests. +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. xMsProperties is user-defined properties to be stored with the +// filesystem, in the format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is +// a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. +// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client filesystemClient) Create(ctx context.Context, filesystem string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemCreateResponse, error) { + if err := validate([]validation{ + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.createPreparer(filesystem, xMsProperties, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemCreateResponse), err +} + +// createPreparer prepares the Create request. +func (client filesystemClient) createPreparer(filesystem string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("PUT", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// createResponder handles the response to the Create request. +func (client filesystemClient) createResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK,http.StatusCreated) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemCreateResponse{rawResponse: resp.Response()}, err +} + +// Delete marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier +// cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem +// with the same identifier will fail with status code 409 (Conflict), with the service returning additional error +// information indicating that the filesystem is being deleted. All other operations, including operations on any files +// or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being +// deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional +// Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. ifModifiedSince is optional. A date and time value. Specify this +// header to perform the operation only if the resource has been modified since the specified date and time. +// ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has not been modified since the specified date and time. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client filesystemClient) Delete(ctx context.Context, filesystem string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemDeleteResponse, error) { + if err := validate([]validation{ + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.deletePreparer(filesystem, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemDeleteResponse), err +} + +// deletePreparer prepares the Delete request. +func (client filesystemClient) deletePreparer(filesystem string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("DELETE", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// deleteResponder handles the response to the Delete request. +func (client filesystemClient) deleteResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK,http.StatusAccepted) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemDeleteResponse{rawResponse: resp.Response()}, err +} + +// GetProperties all system and user-defined filesystem properties are specified in the response headers. +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. xMsClientRequestID is a UUID recorded in the analytics logs for +// troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The period begins when +// the request is received by the service. If the timeout value elapses before the operation completes, the operation +// fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is required when using +// shared key authorization. +func (client filesystemClient) GetProperties(ctx context.Context, filesystem string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemGetPropertiesResponse, error) { + if err := validate([]validation{ + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.getPropertiesPreparer(filesystem, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPropertiesResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemGetPropertiesResponse), err +} + +// getPropertiesPreparer prepares the GetProperties request. +func (client filesystemClient) getPropertiesPreparer(filesystem string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("HEAD", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// getPropertiesResponder handles the response to the GetProperties request. +func (client filesystemClient) getPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemGetPropertiesResponse{rawResponse: resp.Response()}, err +} + +// List list filesystems and their properties in given account. +// +// prefix is filters results to filesystems within the specified prefix. continuation is the number of filesystems +// returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a +// continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in +// the response, it must be specified in a subsequent invocation of the list operation to continue listing the +// filesystems. maxResults is an optional value that specifies the maximum number of items to return. If omitted or +// greater than 5,000, the response will include up to 5,000 items. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client filesystemClient) List(ctx context.Context, prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemList, error) { + if err := validate([]validation{ + { targetValue: maxResults, + constraints: []constraint{ {target: "maxResults", name: null, rule: false , + chain: []constraint{ {target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.listPreparer(prefix, continuation, maxResults, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemList), err +} + +// listPreparer prepares the List request. +func (client filesystemClient) listPreparer(prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("GET", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "account") + if prefix != nil && len(*prefix) > 0 { + params.Set("prefix", *prefix) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if maxResults != nil { + params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// listResponder handles the response to the List request. +func (client filesystemClient) listResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + result:= &FilesystemList{rawResponse: resp.Response()} + if err != nil { + return result, err + } + defer resp.Response().Body.Close() + b, err:= ioutil.ReadAll(resp.Response().Body) + if err != nil { + return result, err + } + if len(b) > 0 { + b = removeBOM(b) + err = json.Unmarshal(b, result) + if err != nil { + return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") + } + } + return result, nil +} + +// SetProperties set properties for the filesystem. This operation supports conditional HTTP requests. For more +// information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. The value must start and end with a letter or number and must contain only +// letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. +// The value must have between 3 and 63 characters. xMsProperties is optional. User-defined properties to be stored +// with the filesystem, in the format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each +// value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character +// set. If the filesystem exists, any properties not included in the list will be removed. All properties are removed +// if the header is omitted. To merge new and existing properties, first get all existing properties and the current +// E-Tag, then make a conditional request with the E-Tag and include values for all properties. ifModifiedSince is +// optional. A date and time value. Specify this header to perform the operation only if the resource has been modified +// since the specified date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to +// perform the operation only if the resource has not been modified since the specified date and time. +// xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client filesystemClient) SetProperties(ctx context.Context, filesystem string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemSetPropertiesResponse, error) { + if err := validate([]validation{ + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.setPropertiesPreparer(filesystem, xMsProperties, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.setPropertiesResponder}, req) + if err != nil { + return nil, err + } + return resp.(*FilesystemSetPropertiesResponse), err +} + +// setPropertiesPreparer prepares the SetProperties request. +func (client filesystemClient) setPropertiesPreparer(filesystem string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("PATCH", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// setPropertiesResponder handles the response to the SetProperties request. +func (client filesystemClient) setPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemSetPropertiesResponse{rawResponse: resp.Response()}, err +} + diff --git a/azbfs/zz_generated_models.go b/azbfs/zz_generated_models.go index b2f8fcb09..734eb9f61 100644 --- a/azbfs/zz_generated_models.go +++ b/azbfs/zz_generated_models.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "reflect" + "strconv" "strings" ) @@ -23,802 +24,945 @@ func joinConst(s interface{}, sep string) string { return strings.Join(ss, sep) } -// CreateFilesystemResponse ... -type CreateFilesystemResponse struct { +func validateError(err error) { + if err != nil { + panic(err) + } +} + +// PathGetPropertiesActionType enumerates the values for path get properties action type. +type PathGetPropertiesActionType string + +const ( + // PathGetPropertiesActionGetAccessControl ... + PathGetPropertiesActionGetAccessControl PathGetPropertiesActionType = "getAccessControl" + // PathGetPropertiesActionGetStatus ... + PathGetPropertiesActionGetStatus PathGetPropertiesActionType = "getStatus" + // PathGetPropertiesActionNone represents an empty PathGetPropertiesActionType. + PathGetPropertiesActionNone PathGetPropertiesActionType = "" +) + +// PossiblePathGetPropertiesActionTypeValues returns an array of possible values for the PathGetPropertiesActionType const type. +func PossiblePathGetPropertiesActionTypeValues() []PathGetPropertiesActionType { + return []PathGetPropertiesActionType{PathGetPropertiesActionGetAccessControl, PathGetPropertiesActionGetStatus, PathGetPropertiesActionNone} +} + +// PathLeaseActionType enumerates the values for path lease action type. +type PathLeaseActionType string + +const ( + // PathLeaseActionAcquire ... + PathLeaseActionAcquire PathLeaseActionType = "acquire" + // PathLeaseActionBreak ... + PathLeaseActionBreak PathLeaseActionType = "break" + // PathLeaseActionChange ... + PathLeaseActionChange PathLeaseActionType = "change" + // PathLeaseActionNone represents an empty PathLeaseActionType. + PathLeaseActionNone PathLeaseActionType = "" + // PathLeaseActionRelease ... + PathLeaseActionRelease PathLeaseActionType = "release" + // PathLeaseActionRenew ... + PathLeaseActionRenew PathLeaseActionType = "renew" +) + +// PossiblePathLeaseActionTypeValues returns an array of possible values for the PathLeaseActionType const type. +func PossiblePathLeaseActionTypeValues() []PathLeaseActionType { + return []PathLeaseActionType{PathLeaseActionAcquire, PathLeaseActionBreak, PathLeaseActionChange, PathLeaseActionNone, PathLeaseActionRelease, PathLeaseActionRenew} +} + +// PathRenameModeType enumerates the values for path rename mode type. +type PathRenameModeType string + +const ( + // PathRenameModeLegacy ... + PathRenameModeLegacy PathRenameModeType = "legacy" + // PathRenameModeNone represents an empty PathRenameModeType. + PathRenameModeNone PathRenameModeType = "" + // PathRenameModePosix ... + PathRenameModePosix PathRenameModeType = "posix" +) + +// PossiblePathRenameModeTypeValues returns an array of possible values for the PathRenameModeType const type. +func PossiblePathRenameModeTypeValues() []PathRenameModeType { + return []PathRenameModeType{PathRenameModeLegacy, PathRenameModeNone, PathRenameModePosix} +} + +// PathResourceType enumerates the values for path resource type. +type PathResourceType string + +const ( + // PathResourceDirectory ... + PathResourceDirectory PathResourceType = "directory" + // PathResourceFile ... + PathResourceFile PathResourceType = "file" + // PathResourceNone represents an empty PathResourceType. + PathResourceNone PathResourceType = "" +) + +// PossiblePathResourceTypeValues returns an array of possible values for the PathResourceType const type. +func PossiblePathResourceTypeValues() []PathResourceType { + return []PathResourceType{PathResourceDirectory, PathResourceFile, PathResourceNone} +} + +// PathUpdateActionType enumerates the values for path update action type. +type PathUpdateActionType string + +const ( + // PathUpdateActionAppend ... + PathUpdateActionAppend PathUpdateActionType = "append" + // PathUpdateActionFlush ... + PathUpdateActionFlush PathUpdateActionType = "flush" + // PathUpdateActionNone represents an empty PathUpdateActionType. + PathUpdateActionNone PathUpdateActionType = "" + // PathUpdateActionSetAccessControl ... + PathUpdateActionSetAccessControl PathUpdateActionType = "setAccessControl" + // PathUpdateActionSetProperties ... + PathUpdateActionSetProperties PathUpdateActionType = "setProperties" +) + +// PossiblePathUpdateActionTypeValues returns an array of possible values for the PathUpdateActionType const type. +func PossiblePathUpdateActionTypeValues() []PathUpdateActionType { + return []PathUpdateActionType{PathUpdateActionAppend, PathUpdateActionFlush, PathUpdateActionNone, PathUpdateActionSetAccessControl, PathUpdateActionSetProperties} +} + +// DataLakeStorageError ... +type DataLakeStorageError struct { + // Error - The service error response object. + Error *DataLakeStorageErrorError `json:"error,omitempty"` +} + +// DataLakeStorageErrorError - The service error response object. +type DataLakeStorageErrorError struct { + // Code - The service error code. + Code *string `json:"code,omitempty"` + // Message - The service error message. + Message *string `json:"message,omitempty"` +} + +// Filesystem ... +type Filesystem struct { + Name *string `json:"name,omitempty"` + LastModified *string `json:"lastModified,omitempty"` + ETag *string `json:"eTag,omitempty"` +} + +// FilesystemCreateResponse ... +type FilesystemCreateResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (cfr CreateFilesystemResponse) Response() *http.Response { - return cfr.rawResponse +func (fcr FilesystemCreateResponse) Response() *http.Response { + return fcr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (cfr CreateFilesystemResponse) StatusCode() int { - return cfr.rawResponse.StatusCode +func (fcr FilesystemCreateResponse) StatusCode() int { + return fcr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (cfr CreateFilesystemResponse) Status() string { - return cfr.rawResponse.Status +func (fcr FilesystemCreateResponse) Status() string { + return fcr.rawResponse.Status } // Date returns the value for header Date. -func (cfr CreateFilesystemResponse) Date() string { - return cfr.rawResponse.Header.Get("Date") +func (fcr FilesystemCreateResponse) Date() string { + return fcr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (cfr CreateFilesystemResponse) ETag() string { - return cfr.rawResponse.Header.Get("ETag") +func (fcr FilesystemCreateResponse) ETag() string { + return fcr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (cfr CreateFilesystemResponse) LastModified() string { - return cfr.rawResponse.Header.Get("Last-Modified") +func (fcr FilesystemCreateResponse) LastModified() string { + return fcr.rawResponse.Header.Get("Last-Modified") } // XMsNamespaceEnabled returns the value for header x-ms-namespace-enabled. -func (cfr CreateFilesystemResponse) XMsNamespaceEnabled() string { - return cfr.rawResponse.Header.Get("x-ms-namespace-enabled") +func (fcr FilesystemCreateResponse) XMsNamespaceEnabled() string { + return fcr.rawResponse.Header.Get("x-ms-namespace-enabled") } // XMsRequestID returns the value for header x-ms-request-id. -func (cfr CreateFilesystemResponse) XMsRequestID() string { - return cfr.rawResponse.Header.Get("x-ms-request-id") +func (fcr FilesystemCreateResponse) XMsRequestID() string { + return fcr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (cfr CreateFilesystemResponse) XMsVersion() string { - return cfr.rawResponse.Header.Get("x-ms-version") +func (fcr FilesystemCreateResponse) XMsVersion() string { + return fcr.rawResponse.Header.Get("x-ms-version") } -// CreatePathResponse ... -type CreatePathResponse struct { +// FilesystemDeleteResponse ... +type FilesystemDeleteResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (cpr CreatePathResponse) Response() *http.Response { - return cpr.rawResponse +func (fdr FilesystemDeleteResponse) Response() *http.Response { + return fdr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (cpr CreatePathResponse) StatusCode() int { - return cpr.rawResponse.StatusCode +func (fdr FilesystemDeleteResponse) StatusCode() int { + return fdr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (cpr CreatePathResponse) Status() string { - return cpr.rawResponse.Status +func (fdr FilesystemDeleteResponse) Status() string { + return fdr.rawResponse.Status } -// ContentLength returns the value for header Content-Length. -func (cpr CreatePathResponse) ContentLength() string { - return cpr.rawResponse.Header.Get("Content-Length") +// Date returns the value for header Date. +func (fdr FilesystemDeleteResponse) Date() string { + return fdr.rawResponse.Header.Get("Date") +} + +// XMsRequestID returns the value for header x-ms-request-id. +func (fdr FilesystemDeleteResponse) XMsRequestID() string { + return fdr.rawResponse.Header.Get("x-ms-request-id") +} + +// XMsVersion returns the value for header x-ms-version. +func (fdr FilesystemDeleteResponse) XMsVersion() string { + return fdr.rawResponse.Header.Get("x-ms-version") +} + +// FilesystemGetPropertiesResponse ... +type FilesystemGetPropertiesResponse struct { + rawResponse *http.Response +} + +// Response returns the raw HTTP response object. +func (fgpr FilesystemGetPropertiesResponse) Response() *http.Response { + return fgpr.rawResponse +} + +// StatusCode returns the HTTP status code of the response, e.g. 200. +func (fgpr FilesystemGetPropertiesResponse) StatusCode() int { + return fgpr.rawResponse.StatusCode +} + +// Status returns the HTTP status message of the response, e.g. "200 OK". +func (fgpr FilesystemGetPropertiesResponse) Status() string { + return fgpr.rawResponse.Status } // Date returns the value for header Date. -func (cpr CreatePathResponse) Date() string { - return cpr.rawResponse.Header.Get("Date") +func (fgpr FilesystemGetPropertiesResponse) Date() string { + return fgpr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (cpr CreatePathResponse) ETag() string { - return cpr.rawResponse.Header.Get("ETag") +func (fgpr FilesystemGetPropertiesResponse) ETag() string { + return fgpr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (cpr CreatePathResponse) LastModified() string { - return cpr.rawResponse.Header.Get("Last-Modified") +func (fgpr FilesystemGetPropertiesResponse) LastModified() string { + return fgpr.rawResponse.Header.Get("Last-Modified") } -// XMsContinuation returns the value for header x-ms-continuation. -func (cpr CreatePathResponse) XMsContinuation() string { - return cpr.rawResponse.Header.Get("x-ms-continuation") +// XMsNamespaceEnabled returns the value for header x-ms-namespace-enabled. +func (fgpr FilesystemGetPropertiesResponse) XMsNamespaceEnabled() string { + return fgpr.rawResponse.Header.Get("x-ms-namespace-enabled") +} + +// XMsProperties returns the value for header x-ms-properties. +func (fgpr FilesystemGetPropertiesResponse) XMsProperties() string { + return fgpr.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (cpr CreatePathResponse) XMsRequestID() string { - return cpr.rawResponse.Header.Get("x-ms-request-id") +func (fgpr FilesystemGetPropertiesResponse) XMsRequestID() string { + return fgpr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (cpr CreatePathResponse) XMsVersion() string { - return cpr.rawResponse.Header.Get("x-ms-version") +func (fgpr FilesystemGetPropertiesResponse) XMsVersion() string { + return fgpr.rawResponse.Header.Get("x-ms-version") } -// DeleteFilesystemResponse ... -type DeleteFilesystemResponse struct { +// FilesystemList ... +type FilesystemList struct { rawResponse *http.Response + Filesystems []Filesystem `json:"filesystems,omitempty"` } // Response returns the raw HTTP response object. -func (dfr DeleteFilesystemResponse) Response() *http.Response { - return dfr.rawResponse +func (fl FilesystemList) Response() *http.Response { + return fl.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (dfr DeleteFilesystemResponse) StatusCode() int { - return dfr.rawResponse.StatusCode +func (fl FilesystemList) StatusCode() int { + return fl.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (dfr DeleteFilesystemResponse) Status() string { - return dfr.rawResponse.Status +func (fl FilesystemList) Status() string { + return fl.rawResponse.Status +} + +// ContentType returns the value for header Content-Type. +func (fl FilesystemList) ContentType() string { + return fl.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (dfr DeleteFilesystemResponse) Date() string { - return dfr.rawResponse.Header.Get("Date") +func (fl FilesystemList) Date() string { + return fl.rawResponse.Header.Get("Date") +} + +// XMsContinuation returns the value for header x-ms-continuation. +func (fl FilesystemList) XMsContinuation() string { + return fl.rawResponse.Header.Get("x-ms-continuation") } // XMsRequestID returns the value for header x-ms-request-id. -func (dfr DeleteFilesystemResponse) XMsRequestID() string { - return dfr.rawResponse.Header.Get("x-ms-request-id") +func (fl FilesystemList) XMsRequestID() string { + return fl.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (dfr DeleteFilesystemResponse) XMsVersion() string { - return dfr.rawResponse.Header.Get("x-ms-version") +func (fl FilesystemList) XMsVersion() string { + return fl.rawResponse.Header.Get("x-ms-version") } -// DeletePathResponse ... -type DeletePathResponse struct { +// FilesystemSetPropertiesResponse ... +type FilesystemSetPropertiesResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (dpr DeletePathResponse) Response() *http.Response { - return dpr.rawResponse +func (fspr FilesystemSetPropertiesResponse) Response() *http.Response { + return fspr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (dpr DeletePathResponse) StatusCode() int { - return dpr.rawResponse.StatusCode +func (fspr FilesystemSetPropertiesResponse) StatusCode() int { + return fspr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (dpr DeletePathResponse) Status() string { - return dpr.rawResponse.Status +func (fspr FilesystemSetPropertiesResponse) Status() string { + return fspr.rawResponse.Status } // Date returns the value for header Date. -func (dpr DeletePathResponse) Date() string { - return dpr.rawResponse.Header.Get("Date") +func (fspr FilesystemSetPropertiesResponse) Date() string { + return fspr.rawResponse.Header.Get("Date") } -// XMsContinuation returns the value for header x-ms-continuation. -func (dpr DeletePathResponse) XMsContinuation() string { - return dpr.rawResponse.Header.Get("x-ms-continuation") +// ETag returns the value for header ETag. +func (fspr FilesystemSetPropertiesResponse) ETag() string { + return fspr.rawResponse.Header.Get("ETag") } -// XMsRequestID returns the value for header x-ms-request-id. -func (dpr DeletePathResponse) XMsRequestID() string { - return dpr.rawResponse.Header.Get("x-ms-request-id") +// LastModified returns the value for header Last-Modified. +func (fspr FilesystemSetPropertiesResponse) LastModified() string { + return fspr.rawResponse.Header.Get("Last-Modified") } -// XMsVersion returns the value for header x-ms-version. -func (dpr DeletePathResponse) XMsVersion() string { - return dpr.rawResponse.Header.Get("x-ms-version") +// XMsRequestID returns the value for header x-ms-request-id. +func (fspr FilesystemSetPropertiesResponse) XMsRequestID() string { + return fspr.rawResponse.Header.Get("x-ms-request-id") } -// ErrorSchema ... -type ErrorSchema struct { - // Error - The service error response object. - Error *ErrorSchemaError `json:"error,omitempty"` +// XMsVersion returns the value for header x-ms-version. +func (fspr FilesystemSetPropertiesResponse) XMsVersion() string { + return fspr.rawResponse.Header.Get("x-ms-version") } -// ErrorSchemaError - The service error response object. -type ErrorSchemaError struct { - // Code - The service error code. - Code *string `json:"code,omitempty"` - // Message - The service error message. - Message *string `json:"message,omitempty"` +// Path ... +type Path struct { + Name *string `json:"name,omitempty"` + IsDirectory *bool `json:"isDirectory,omitempty"` + LastModified *string `json:"lastModified,omitempty"` + ETag *string `json:"eTag,omitempty"` + ContentLength *int64 `json:"contentLength,omitempty"` + Owner *string `json:"owner,omitempty"` + Group *string `json:"group,omitempty"` + Permissions *string `json:"permissions,omitempty"` } -// GetFilesystemPropertiesResponse ... -type GetFilesystemPropertiesResponse struct { +// PathCreateResponse ... +type PathCreateResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (gfpr GetFilesystemPropertiesResponse) Response() *http.Response { - return gfpr.rawResponse +func (pcr PathCreateResponse) Response() *http.Response { + return pcr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (gfpr GetFilesystemPropertiesResponse) StatusCode() int { - return gfpr.rawResponse.StatusCode +func (pcr PathCreateResponse) StatusCode() int { + return pcr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (gfpr GetFilesystemPropertiesResponse) Status() string { - return gfpr.rawResponse.Status +func (pcr PathCreateResponse) Status() string { + return pcr.rawResponse.Status +} + +// ContentLength returns the value for header Content-Length. +func (pcr PathCreateResponse) ContentLength() int64 { + s := pcr.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i } // Date returns the value for header Date. -func (gfpr GetFilesystemPropertiesResponse) Date() string { - return gfpr.rawResponse.Header.Get("Date") +func (pcr PathCreateResponse) Date() string { + return pcr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (gfpr GetFilesystemPropertiesResponse) ETag() string { - return gfpr.rawResponse.Header.Get("ETag") +func (pcr PathCreateResponse) ETag() string { + return pcr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (gfpr GetFilesystemPropertiesResponse) LastModified() string { - return gfpr.rawResponse.Header.Get("Last-Modified") +func (pcr PathCreateResponse) LastModified() string { + return pcr.rawResponse.Header.Get("Last-Modified") } -// XMsNamespaceEnabled returns the value for header x-ms-namespace-enabled. -func (gfpr GetFilesystemPropertiesResponse) XMsNamespaceEnabled() string { - return gfpr.rawResponse.Header.Get("x-ms-namespace-enabled") +// XMsContinuation returns the value for header x-ms-continuation. +func (pcr PathCreateResponse) XMsContinuation() string { + return pcr.rawResponse.Header.Get("x-ms-continuation") } -// XMsProperties returns the value for header x-ms-properties. -func (gfpr GetFilesystemPropertiesResponse) XMsProperties() string { - return gfpr.rawResponse.Header.Get("x-ms-properties") +// XMsRequestID returns the value for header x-ms-request-id. +func (pcr PathCreateResponse) XMsRequestID() string { + return pcr.rawResponse.Header.Get("x-ms-request-id") +} + +// XMsVersion returns the value for header x-ms-version. +func (pcr PathCreateResponse) XMsVersion() string { + return pcr.rawResponse.Header.Get("x-ms-version") +} + +// PathDeleteResponse ... +type PathDeleteResponse struct { + rawResponse *http.Response +} + +// Response returns the raw HTTP response object. +func (pdr PathDeleteResponse) Response() *http.Response { + return pdr.rawResponse +} + +// StatusCode returns the HTTP status code of the response, e.g. 200. +func (pdr PathDeleteResponse) StatusCode() int { + return pdr.rawResponse.StatusCode +} + +// Status returns the HTTP status message of the response, e.g. "200 OK". +func (pdr PathDeleteResponse) Status() string { + return pdr.rawResponse.Status +} + +// Date returns the value for header Date. +func (pdr PathDeleteResponse) Date() string { + return pdr.rawResponse.Header.Get("Date") +} + +// XMsContinuation returns the value for header x-ms-continuation. +func (pdr PathDeleteResponse) XMsContinuation() string { + return pdr.rawResponse.Header.Get("x-ms-continuation") } // XMsRequestID returns the value for header x-ms-request-id. -func (gfpr GetFilesystemPropertiesResponse) XMsRequestID() string { - return gfpr.rawResponse.Header.Get("x-ms-request-id") +func (pdr PathDeleteResponse) XMsRequestID() string { + return pdr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (gfpr GetFilesystemPropertiesResponse) XMsVersion() string { - return gfpr.rawResponse.Header.Get("x-ms-version") +func (pdr PathDeleteResponse) XMsVersion() string { + return pdr.rawResponse.Header.Get("x-ms-version") } -// GetPathPropertiesResponse ... -type GetPathPropertiesResponse struct { +// PathGetPropertiesResponse ... +type PathGetPropertiesResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (gppr GetPathPropertiesResponse) Response() *http.Response { - return gppr.rawResponse +func (pgpr PathGetPropertiesResponse) Response() *http.Response { + return pgpr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (gppr GetPathPropertiesResponse) StatusCode() int { - return gppr.rawResponse.StatusCode +func (pgpr PathGetPropertiesResponse) StatusCode() int { + return pgpr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (gppr GetPathPropertiesResponse) Status() string { - return gppr.rawResponse.Status +func (pgpr PathGetPropertiesResponse) Status() string { + return pgpr.rawResponse.Status } // AcceptRanges returns the value for header Accept-Ranges. -func (gppr GetPathPropertiesResponse) AcceptRanges() string { - return gppr.rawResponse.Header.Get("Accept-Ranges") +func (pgpr PathGetPropertiesResponse) AcceptRanges() string { + return pgpr.rawResponse.Header.Get("Accept-Ranges") } // CacheControl returns the value for header Cache-Control. -func (gppr GetPathPropertiesResponse) CacheControl() string { - return gppr.rawResponse.Header.Get("Cache-Control") +func (pgpr PathGetPropertiesResponse) CacheControl() string { + return pgpr.rawResponse.Header.Get("Cache-Control") } // ContentDisposition returns the value for header Content-Disposition. -func (gppr GetPathPropertiesResponse) ContentDisposition() string { - return gppr.rawResponse.Header.Get("Content-Disposition") +func (pgpr PathGetPropertiesResponse) ContentDisposition() string { + return pgpr.rawResponse.Header.Get("Content-Disposition") } // ContentEncoding returns the value for header Content-Encoding. -func (gppr GetPathPropertiesResponse) ContentEncoding() string { - return gppr.rawResponse.Header.Get("Content-Encoding") +func (pgpr PathGetPropertiesResponse) ContentEncoding() string { + return pgpr.rawResponse.Header.Get("Content-Encoding") } // ContentLanguage returns the value for header Content-Language. -func (gppr GetPathPropertiesResponse) ContentLanguage() string { - return gppr.rawResponse.Header.Get("Content-Language") +func (pgpr PathGetPropertiesResponse) ContentLanguage() string { + return pgpr.rawResponse.Header.Get("Content-Language") } // ContentLength returns the value for header Content-Length. -func (gppr GetPathPropertiesResponse) ContentLength() string { - return gppr.rawResponse.Header.Get("Content-Length") +func (pgpr PathGetPropertiesResponse) ContentLength() int64 { + s := pgpr.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i +} + +// ContentMD5 returns the value for header Content-MD5. +func (pgpr PathGetPropertiesResponse) ContentMD5() string { + return pgpr.rawResponse.Header.Get("Content-MD5") } // ContentRange returns the value for header Content-Range. -func (gppr GetPathPropertiesResponse) ContentRange() string { - return gppr.rawResponse.Header.Get("Content-Range") +func (pgpr PathGetPropertiesResponse) ContentRange() string { + return pgpr.rawResponse.Header.Get("Content-Range") } // ContentType returns the value for header Content-Type. -func (gppr GetPathPropertiesResponse) ContentType() string { - return gppr.rawResponse.Header.Get("Content-Type") +func (pgpr PathGetPropertiesResponse) ContentType() string { + return pgpr.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (gppr GetPathPropertiesResponse) Date() string { - return gppr.rawResponse.Header.Get("Date") +func (pgpr PathGetPropertiesResponse) Date() string { + return pgpr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (gppr GetPathPropertiesResponse) ETag() string { - return gppr.rawResponse.Header.Get("ETag") +func (pgpr PathGetPropertiesResponse) ETag() string { + return pgpr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (gppr GetPathPropertiesResponse) LastModified() string { - return gppr.rawResponse.Header.Get("Last-Modified") +func (pgpr PathGetPropertiesResponse) LastModified() string { + return pgpr.rawResponse.Header.Get("Last-Modified") } // XMsACL returns the value for header x-ms-acl. -func (gppr GetPathPropertiesResponse) XMsACL() string { - return gppr.rawResponse.Header.Get("x-ms-acl") +func (pgpr PathGetPropertiesResponse) XMsACL() string { + return pgpr.rawResponse.Header.Get("x-ms-acl") } // XMsGroup returns the value for header x-ms-group. -func (gppr GetPathPropertiesResponse) XMsGroup() string { - return gppr.rawResponse.Header.Get("x-ms-group") +func (pgpr PathGetPropertiesResponse) XMsGroup() string { + return pgpr.rawResponse.Header.Get("x-ms-group") } // XMsLeaseDuration returns the value for header x-ms-lease-duration. -func (gppr GetPathPropertiesResponse) XMsLeaseDuration() string { - return gppr.rawResponse.Header.Get("x-ms-lease-duration") +func (pgpr PathGetPropertiesResponse) XMsLeaseDuration() string { + return pgpr.rawResponse.Header.Get("x-ms-lease-duration") } // XMsLeaseState returns the value for header x-ms-lease-state. -func (gppr GetPathPropertiesResponse) XMsLeaseState() string { - return gppr.rawResponse.Header.Get("x-ms-lease-state") +func (pgpr PathGetPropertiesResponse) XMsLeaseState() string { + return pgpr.rawResponse.Header.Get("x-ms-lease-state") } // XMsLeaseStatus returns the value for header x-ms-lease-status. -func (gppr GetPathPropertiesResponse) XMsLeaseStatus() string { - return gppr.rawResponse.Header.Get("x-ms-lease-status") +func (pgpr PathGetPropertiesResponse) XMsLeaseStatus() string { + return pgpr.rawResponse.Header.Get("x-ms-lease-status") } // XMsOwner returns the value for header x-ms-owner. -func (gppr GetPathPropertiesResponse) XMsOwner() string { - return gppr.rawResponse.Header.Get("x-ms-owner") +func (pgpr PathGetPropertiesResponse) XMsOwner() string { + return pgpr.rawResponse.Header.Get("x-ms-owner") } // XMsPermissions returns the value for header x-ms-permissions. -func (gppr GetPathPropertiesResponse) XMsPermissions() string { - return gppr.rawResponse.Header.Get("x-ms-permissions") +func (pgpr PathGetPropertiesResponse) XMsPermissions() string { + return pgpr.rawResponse.Header.Get("x-ms-permissions") } // XMsProperties returns the value for header x-ms-properties. -func (gppr GetPathPropertiesResponse) XMsProperties() string { - return gppr.rawResponse.Header.Get("x-ms-properties") +func (pgpr PathGetPropertiesResponse) XMsProperties() string { + return pgpr.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (gppr GetPathPropertiesResponse) XMsRequestID() string { - return gppr.rawResponse.Header.Get("x-ms-request-id") +func (pgpr PathGetPropertiesResponse) XMsRequestID() string { + return pgpr.rawResponse.Header.Get("x-ms-request-id") } // XMsResourceType returns the value for header x-ms-resource-type. -func (gppr GetPathPropertiesResponse) XMsResourceType() string { - return gppr.rawResponse.Header.Get("x-ms-resource-type") +func (pgpr PathGetPropertiesResponse) XMsResourceType() string { + return pgpr.rawResponse.Header.Get("x-ms-resource-type") } // XMsVersion returns the value for header x-ms-version. -func (gppr GetPathPropertiesResponse) XMsVersion() string { - return gppr.rawResponse.Header.Get("x-ms-version") +func (pgpr PathGetPropertiesResponse) XMsVersion() string { + return pgpr.rawResponse.Header.Get("x-ms-version") } -// LeasePathResponse ... -type LeasePathResponse struct { +// PathLeaseResponse ... +type PathLeaseResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (lpr LeasePathResponse) Response() *http.Response { - return lpr.rawResponse +func (plr PathLeaseResponse) Response() *http.Response { + return plr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (lpr LeasePathResponse) StatusCode() int { - return lpr.rawResponse.StatusCode +func (plr PathLeaseResponse) StatusCode() int { + return plr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (lpr LeasePathResponse) Status() string { - return lpr.rawResponse.Status +func (plr PathLeaseResponse) Status() string { + return plr.rawResponse.Status } // Date returns the value for header Date. -func (lpr LeasePathResponse) Date() string { - return lpr.rawResponse.Header.Get("Date") +func (plr PathLeaseResponse) Date() string { + return plr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (lpr LeasePathResponse) ETag() string { - return lpr.rawResponse.Header.Get("ETag") +func (plr PathLeaseResponse) ETag() string { + return plr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (lpr LeasePathResponse) LastModified() string { - return lpr.rawResponse.Header.Get("Last-Modified") +func (plr PathLeaseResponse) LastModified() string { + return plr.rawResponse.Header.Get("Last-Modified") } // XMsLeaseID returns the value for header x-ms-lease-id. -func (lpr LeasePathResponse) XMsLeaseID() string { - return lpr.rawResponse.Header.Get("x-ms-lease-id") +func (plr PathLeaseResponse) XMsLeaseID() string { + return plr.rawResponse.Header.Get("x-ms-lease-id") } // XMsLeaseTime returns the value for header x-ms-lease-time. -func (lpr LeasePathResponse) XMsLeaseTime() string { - return lpr.rawResponse.Header.Get("x-ms-lease-time") +func (plr PathLeaseResponse) XMsLeaseTime() string { + return plr.rawResponse.Header.Get("x-ms-lease-time") } // XMsRequestID returns the value for header x-ms-request-id. -func (lpr LeasePathResponse) XMsRequestID() string { - return lpr.rawResponse.Header.Get("x-ms-request-id") +func (plr PathLeaseResponse) XMsRequestID() string { + return plr.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (lpr LeasePathResponse) XMsVersion() string { - return lpr.rawResponse.Header.Get("x-ms-version") -} - -// ListEntrySchema ... -type ListEntrySchema struct { - Name *string `json:"name,omitempty"` - IsDirectory *bool `json:"isDirectory,string,omitempty"` - LastModified *string `json:"lastModified,omitempty"` - ETag *string `json:"eTag,omitempty"` - ContentLength *int64 `json:"contentLength,string,omitempty"` - Owner *string `json:"owner,omitempty"` - Group *string `json:"group,omitempty"` - Permissions *string `json:"permissions,omitempty"` -} - -// ListFilesystemEntry ... -type ListFilesystemEntry struct { - Name *string `json:"name,omitempty"` - LastModified *string `json:"lastModified,omitempty"` - ETag *string `json:"eTag,omitempty"` +func (plr PathLeaseResponse) XMsVersion() string { + return plr.rawResponse.Header.Get("x-ms-version") } -// ListFilesystemSchema ... -type ListFilesystemSchema struct { +// PathList ... +type PathList struct { rawResponse *http.Response - Filesystems []ListFilesystemEntry `json:"filesystems,omitempty"` + Paths []Path `json:"paths,omitempty"` } // Response returns the raw HTTP response object. -func (lfs ListFilesystemSchema) Response() *http.Response { - return lfs.rawResponse +func (pl PathList) Response() *http.Response { + return pl.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (lfs ListFilesystemSchema) StatusCode() int { - return lfs.rawResponse.StatusCode +func (pl PathList) StatusCode() int { + return pl.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (lfs ListFilesystemSchema) Status() string { - return lfs.rawResponse.Status -} - -// ContentType returns the value for header Content-Type. -func (lfs ListFilesystemSchema) ContentType() string { - return lfs.rawResponse.Header.Get("Content-Type") +func (pl PathList) Status() string { + return pl.rawResponse.Status } // Date returns the value for header Date. -func (lfs ListFilesystemSchema) Date() string { - return lfs.rawResponse.Header.Get("Date") -} - -// XMsContinuation returns the value for header x-ms-continuation. -func (lfs ListFilesystemSchema) XMsContinuation() string { - return lfs.rawResponse.Header.Get("x-ms-continuation") -} - -// XMsRequestID returns the value for header x-ms-request-id. -func (lfs ListFilesystemSchema) XMsRequestID() string { - return lfs.rawResponse.Header.Get("x-ms-request-id") -} - -// XMsVersion returns the value for header x-ms-version. -func (lfs ListFilesystemSchema) XMsVersion() string { - return lfs.rawResponse.Header.Get("x-ms-version") -} - -// ListSchema ... -type ListSchema struct { - rawResponse *http.Response - Paths []ListEntrySchema `json:"paths,omitempty"` -} - -// Response returns the raw HTTP response object. -func (ls ListSchema) Response() *http.Response { - return ls.rawResponse -} - -// StatusCode returns the HTTP status code of the response, e.g. 200. -func (ls ListSchema) StatusCode() int { - return ls.rawResponse.StatusCode -} - -// Status returns the HTTP status message of the response, e.g. "200 OK". -func (ls ListSchema) Status() string { - return ls.rawResponse.Status -} - -// Date returns the value for header Date. -func (ls ListSchema) Date() string { - return ls.rawResponse.Header.Get("Date") +func (pl PathList) Date() string { + return pl.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (ls ListSchema) ETag() string { - return ls.rawResponse.Header.Get("ETag") +func (pl PathList) ETag() string { + return pl.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (ls ListSchema) LastModified() string { - return ls.rawResponse.Header.Get("Last-Modified") +func (pl PathList) LastModified() string { + return pl.rawResponse.Header.Get("Last-Modified") } // XMsContinuation returns the value for header x-ms-continuation. -func (ls ListSchema) XMsContinuation() string { - return ls.rawResponse.Header.Get("x-ms-continuation") +func (pl PathList) XMsContinuation() string { + return pl.rawResponse.Header.Get("x-ms-continuation") } // XMsRequestID returns the value for header x-ms-request-id. -func (ls ListSchema) XMsRequestID() string { - return ls.rawResponse.Header.Get("x-ms-request-id") +func (pl PathList) XMsRequestID() string { + return pl.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (ls ListSchema) XMsVersion() string { - return ls.rawResponse.Header.Get("x-ms-version") +func (pl PathList) XMsVersion() string { + return pl.rawResponse.Header.Get("x-ms-version") } -// ReadPathResponse ... -type ReadPathResponse struct { +// PathUpdateResponse ... +type PathUpdateResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (rpr ReadPathResponse) Response() *http.Response { - return rpr.rawResponse +func (pur PathUpdateResponse) Response() *http.Response { + return pur.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (rpr ReadPathResponse) StatusCode() int { - return rpr.rawResponse.StatusCode +func (pur PathUpdateResponse) StatusCode() int { + return pur.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (rpr ReadPathResponse) Status() string { - return rpr.rawResponse.Status -} - -// Body returns the raw HTTP response object's Body. -func (rpr ReadPathResponse) Body() io.ReadCloser { - return rpr.rawResponse.Body +func (pur PathUpdateResponse) Status() string { + return pur.rawResponse.Status } // AcceptRanges returns the value for header Accept-Ranges. -func (rpr ReadPathResponse) AcceptRanges() string { - return rpr.rawResponse.Header.Get("Accept-Ranges") +func (pur PathUpdateResponse) AcceptRanges() string { + return pur.rawResponse.Header.Get("Accept-Ranges") } // CacheControl returns the value for header Cache-Control. -func (rpr ReadPathResponse) CacheControl() string { - return rpr.rawResponse.Header.Get("Cache-Control") +func (pur PathUpdateResponse) CacheControl() string { + return pur.rawResponse.Header.Get("Cache-Control") } // ContentDisposition returns the value for header Content-Disposition. -func (rpr ReadPathResponse) ContentDisposition() string { - return rpr.rawResponse.Header.Get("Content-Disposition") +func (pur PathUpdateResponse) ContentDisposition() string { + return pur.rawResponse.Header.Get("Content-Disposition") } // ContentEncoding returns the value for header Content-Encoding. -func (rpr ReadPathResponse) ContentEncoding() string { - return rpr.rawResponse.Header.Get("Content-Encoding") +func (pur PathUpdateResponse) ContentEncoding() string { + return pur.rawResponse.Header.Get("Content-Encoding") } // ContentLanguage returns the value for header Content-Language. -func (rpr ReadPathResponse) ContentLanguage() string { - return rpr.rawResponse.Header.Get("Content-Language") +func (pur PathUpdateResponse) ContentLanguage() string { + return pur.rawResponse.Header.Get("Content-Language") } // ContentLength returns the value for header Content-Length. -func (rpr ReadPathResponse) ContentLength() string { - return rpr.rawResponse.Header.Get("Content-Length") +func (pur PathUpdateResponse) ContentLength() int64 { + s := pur.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i } // ContentRange returns the value for header Content-Range. -func (rpr ReadPathResponse) ContentRange() string { - return rpr.rawResponse.Header.Get("Content-Range") +func (pur PathUpdateResponse) ContentRange() string { + return pur.rawResponse.Header.Get("Content-Range") } // ContentType returns the value for header Content-Type. -func (rpr ReadPathResponse) ContentType() string { - return rpr.rawResponse.Header.Get("Content-Type") +func (pur PathUpdateResponse) ContentType() string { + return pur.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (rpr ReadPathResponse) Date() string { - return rpr.rawResponse.Header.Get("Date") +func (pur PathUpdateResponse) Date() string { + return pur.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (rpr ReadPathResponse) ETag() string { - return rpr.rawResponse.Header.Get("ETag") +func (pur PathUpdateResponse) ETag() string { + return pur.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (rpr ReadPathResponse) LastModified() string { - return rpr.rawResponse.Header.Get("Last-Modified") -} - -// XMsLeaseDuration returns the value for header x-ms-lease-duration. -func (rpr ReadPathResponse) XMsLeaseDuration() string { - return rpr.rawResponse.Header.Get("x-ms-lease-duration") -} - -// XMsLeaseState returns the value for header x-ms-lease-state. -func (rpr ReadPathResponse) XMsLeaseState() string { - return rpr.rawResponse.Header.Get("x-ms-lease-state") -} - -// XMsLeaseStatus returns the value for header x-ms-lease-status. -func (rpr ReadPathResponse) XMsLeaseStatus() string { - return rpr.rawResponse.Header.Get("x-ms-lease-status") +func (pur PathUpdateResponse) LastModified() string { + return pur.rawResponse.Header.Get("Last-Modified") } // XMsProperties returns the value for header x-ms-properties. -func (rpr ReadPathResponse) XMsProperties() string { - return rpr.rawResponse.Header.Get("x-ms-properties") +func (pur PathUpdateResponse) XMsProperties() string { + return pur.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (rpr ReadPathResponse) XMsRequestID() string { - return rpr.rawResponse.Header.Get("x-ms-request-id") -} - -// XMsResourceType returns the value for header x-ms-resource-type. -func (rpr ReadPathResponse) XMsResourceType() string { - return rpr.rawResponse.Header.Get("x-ms-resource-type") +func (pur PathUpdateResponse) XMsRequestID() string { + return pur.rawResponse.Header.Get("x-ms-request-id") } // XMsVersion returns the value for header x-ms-version. -func (rpr ReadPathResponse) XMsVersion() string { - return rpr.rawResponse.Header.Get("x-ms-version") +func (pur PathUpdateResponse) XMsVersion() string { + return pur.rawResponse.Header.Get("x-ms-version") } -// SetFilesystemPropertiesResponse ... -type SetFilesystemPropertiesResponse struct { +// ReadResponse - Wraps the response from the pathClient.Read method. +type ReadResponse struct { rawResponse *http.Response } // Response returns the raw HTTP response object. -func (sfpr SetFilesystemPropertiesResponse) Response() *http.Response { - return sfpr.rawResponse +func (rr ReadResponse) Response() *http.Response { + return rr.rawResponse } // StatusCode returns the HTTP status code of the response, e.g. 200. -func (sfpr SetFilesystemPropertiesResponse) StatusCode() int { - return sfpr.rawResponse.StatusCode +func (rr ReadResponse) StatusCode() int { + return rr.rawResponse.StatusCode } // Status returns the HTTP status message of the response, e.g. "200 OK". -func (sfpr SetFilesystemPropertiesResponse) Status() string { - return sfpr.rawResponse.Status -} - -// Date returns the value for header Date. -func (sfpr SetFilesystemPropertiesResponse) Date() string { - return sfpr.rawResponse.Header.Get("Date") -} - -// ETag returns the value for header ETag. -func (sfpr SetFilesystemPropertiesResponse) ETag() string { - return sfpr.rawResponse.Header.Get("ETag") -} - -// LastModified returns the value for header Last-Modified. -func (sfpr SetFilesystemPropertiesResponse) LastModified() string { - return sfpr.rawResponse.Header.Get("Last-Modified") -} - -// XMsRequestID returns the value for header x-ms-request-id. -func (sfpr SetFilesystemPropertiesResponse) XMsRequestID() string { - return sfpr.rawResponse.Header.Get("x-ms-request-id") -} - -// XMsVersion returns the value for header x-ms-version. -func (sfpr SetFilesystemPropertiesResponse) XMsVersion() string { - return sfpr.rawResponse.Header.Get("x-ms-version") +func (rr ReadResponse) Status() string { + return rr.rawResponse.Status } -// UpdatePathResponse ... -type UpdatePathResponse struct { - rawResponse *http.Response -} - -// Response returns the raw HTTP response object. -func (upr UpdatePathResponse) Response() *http.Response { - return upr.rawResponse -} - -// StatusCode returns the HTTP status code of the response, e.g. 200. -func (upr UpdatePathResponse) StatusCode() int { - return upr.rawResponse.StatusCode -} - -// Status returns the HTTP status message of the response, e.g. "200 OK". -func (upr UpdatePathResponse) Status() string { - return upr.rawResponse.Status +// Body returns the raw HTTP response object's Body. +func (rr ReadResponse) Body() io.ReadCloser { + return rr.rawResponse.Body } // AcceptRanges returns the value for header Accept-Ranges. -func (upr UpdatePathResponse) AcceptRanges() string { - return upr.rawResponse.Header.Get("Accept-Ranges") +func (rr ReadResponse) AcceptRanges() string { + return rr.rawResponse.Header.Get("Accept-Ranges") } // CacheControl returns the value for header Cache-Control. -func (upr UpdatePathResponse) CacheControl() string { - return upr.rawResponse.Header.Get("Cache-Control") +func (rr ReadResponse) CacheControl() string { + return rr.rawResponse.Header.Get("Cache-Control") } // ContentDisposition returns the value for header Content-Disposition. -func (upr UpdatePathResponse) ContentDisposition() string { - return upr.rawResponse.Header.Get("Content-Disposition") +func (rr ReadResponse) ContentDisposition() string { + return rr.rawResponse.Header.Get("Content-Disposition") } // ContentEncoding returns the value for header Content-Encoding. -func (upr UpdatePathResponse) ContentEncoding() string { - return upr.rawResponse.Header.Get("Content-Encoding") +func (rr ReadResponse) ContentEncoding() string { + return rr.rawResponse.Header.Get("Content-Encoding") } // ContentLanguage returns the value for header Content-Language. -func (upr UpdatePathResponse) ContentLanguage() string { - return upr.rawResponse.Header.Get("Content-Language") +func (rr ReadResponse) ContentLanguage() string { + return rr.rawResponse.Header.Get("Content-Language") } // ContentLength returns the value for header Content-Length. -func (upr UpdatePathResponse) ContentLength() string { - return upr.rawResponse.Header.Get("Content-Length") +func (rr ReadResponse) ContentLength() int64 { + s := rr.rawResponse.Header.Get("Content-Length") + if s == "" { + return -1 + } + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + i = 0 + } + return i +} + +// ContentMD5 returns the value for header Content-MD5. +func (rr ReadResponse) ContentMD5() string { + return rr.rawResponse.Header.Get("Content-MD5") } // ContentRange returns the value for header Content-Range. -func (upr UpdatePathResponse) ContentRange() string { - return upr.rawResponse.Header.Get("Content-Range") +func (rr ReadResponse) ContentRange() string { + return rr.rawResponse.Header.Get("Content-Range") } // ContentType returns the value for header Content-Type. -func (upr UpdatePathResponse) ContentType() string { - return upr.rawResponse.Header.Get("Content-Type") +func (rr ReadResponse) ContentType() string { + return rr.rawResponse.Header.Get("Content-Type") } // Date returns the value for header Date. -func (upr UpdatePathResponse) Date() string { - return upr.rawResponse.Header.Get("Date") +func (rr ReadResponse) Date() string { + return rr.rawResponse.Header.Get("Date") } // ETag returns the value for header ETag. -func (upr UpdatePathResponse) ETag() string { - return upr.rawResponse.Header.Get("ETag") +func (rr ReadResponse) ETag() string { + return rr.rawResponse.Header.Get("ETag") } // LastModified returns the value for header Last-Modified. -func (upr UpdatePathResponse) LastModified() string { - return upr.rawResponse.Header.Get("Last-Modified") +func (rr ReadResponse) LastModified() string { + return rr.rawResponse.Header.Get("Last-Modified") +} + +// XMsLeaseDuration returns the value for header x-ms-lease-duration. +func (rr ReadResponse) XMsLeaseDuration() string { + return rr.rawResponse.Header.Get("x-ms-lease-duration") +} + +// XMsLeaseState returns the value for header x-ms-lease-state. +func (rr ReadResponse) XMsLeaseState() string { + return rr.rawResponse.Header.Get("x-ms-lease-state") +} + +// XMsLeaseStatus returns the value for header x-ms-lease-status. +func (rr ReadResponse) XMsLeaseStatus() string { + return rr.rawResponse.Header.Get("x-ms-lease-status") } // XMsProperties returns the value for header x-ms-properties. -func (upr UpdatePathResponse) XMsProperties() string { - return upr.rawResponse.Header.Get("x-ms-properties") +func (rr ReadResponse) XMsProperties() string { + return rr.rawResponse.Header.Get("x-ms-properties") } // XMsRequestID returns the value for header x-ms-request-id. -func (upr UpdatePathResponse) XMsRequestID() string { - return upr.rawResponse.Header.Get("x-ms-request-id") +func (rr ReadResponse) XMsRequestID() string { + return rr.rawResponse.Header.Get("x-ms-request-id") +} + +// XMsResourceType returns the value for header x-ms-resource-type. +func (rr ReadResponse) XMsResourceType() string { + return rr.rawResponse.Header.Get("x-ms-resource-type") } // XMsVersion returns the value for header x-ms-version. -func (upr UpdatePathResponse) XMsVersion() string { - return upr.rawResponse.Header.Get("x-ms-version") +func (rr ReadResponse) XMsVersion() string { + return rr.rawResponse.Header.Get("x-ms-version") } diff --git a/azbfs/zz_generated_path.go b/azbfs/zz_generated_path.go new file mode 100644 index 000000000..51360fd17 --- /dev/null +++ b/azbfs/zz_generated_path.go @@ -0,0 +1,1009 @@ +package azbfs + +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +import ( + "net/url" + "github.com/Azure/azure-pipeline-go/pipeline" + "net/url" + "net/http" + "net/url" + "context" + "net/url" + "strconv" + "net/url" + "io" + "net/url" + "encoding/json" + "net/url" + "io/ioutil" +) + +// pathClient is the azure Data Lake Storage provides storage for Hadoop and other big data workloads. +type pathClient struct { + managementClient +} +// newPathClient creates an instance of the pathClient client. +func newPathClient(url url.URL, p pipeline.Pipeline) pathClient { + return pathClient{newManagementClient(url, p)} +} + +// Create create or rename a file or directory. By default, the destination is overwritten and if the destination +// already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more +// information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// To fail if the destination already exists, use a conditional request with If-None-Match: "*". +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. resource is required only for +// Create File and Create Directory. The value must be "file" or "directory". continuation is optional. When renaming +// a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be +// renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is +// returned in the response, it must be specified in a subsequent invocation of the rename operation to continue +// renaming the directory. mode is optional. Valid only when namespace is enabled. This parameter determines the +// behavior of the rename operation. The value must be "legacy" or "posix", and the default value will be "posix". +// cacheControl is optional. The service stores this value and includes it in the "Cache-Control" response header for +// "Read File" operations for "Read File" operations. contentEncoding is optional. Specifies which content encodings +// have been applied to the file. This value is returned to the client when the "Read File" operation is performed. +// contentLanguage is optional. Specifies the natural language used by the intended audience for the file. +// contentDisposition is optional. The service stores this value and includes it in the "Content-Disposition" response +// header for "Read File" operations. xMsCacheControl is optional. The service stores this value and includes it in +// the "Cache-Control" response header for "Read File" operations. xMsContentType is optional. The service stores this +// value and includes it in the "Content-Type" response header for "Read File" operations. xMsContentEncoding is +// optional. The service stores this value and includes it in the "Content-Encoding" response header for "Read File" +// operations. xMsContentLanguage is optional. The service stores this value and includes it in the "Content-Language" +// response header for "Read File" operations. xMsContentDisposition is optional. The service stores this value and +// includes it in the "Content-Disposition" response header for "Read File" operations. xMsRenameSource is an optional +// file or directory to be renamed. The value must have the following format: "/{filesystem}/{path}". If +// "x-ms-properties" is specified, the properties will overwrite the existing properties; otherwise, the existing +// properties will be preserved. This value must be a URL percent-encoded string. Note that the string may only contain +// ASCII characters in the ISO-8859-1 character set. xMsLeaseID is optional. A lease ID for the path specified in the +// URI. The path to be overwritten must have an active lease and the lease ID must match. xMsSourceLeaseID is optional +// for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID +// must match. xMsProperties is optional. User-defined properties to be stored with the file or directory, in the +// format of a comma-separated list of name and value pairs "n1=v1, n2=v2, ...", where each value is a base64 encoded +// string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. xMsPermissions is +// optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the +// file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The +// sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. +// xMsUmask is optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or +// directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or +// directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. +// For example, if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is 0777 for +// a directory and 0666 for a file. The default umask is 0027. The umask must be specified in 4-digit octal notation +// (e.g. 0766). ifMatch is optional. An ETag value. Specify this header to perform the operation only if the +// resource's ETag matches the value specified. The ETag must be specified in quotes. ifNoneMatch is optional. An ETag +// value or the special wildcard ("*") value. Specify this header to perform the operation only if the resource's ETag +// does not match the value specified. The ETag must be specified in quotes. ifModifiedSince is optional. A date and +// time value. Specify this header to perform the operation only if the resource has been modified since the specified +// date and time. ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation +// only if the resource has not been modified since the specified date and time. xMsSourceIfMatch is optional. An ETag +// value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. +// The ETag must be specified in quotes. xMsSourceIfNoneMatch is optional. An ETag value or the special wildcard ("*") +// value. Specify this header to perform the rename operation only if the source's ETag does not match the value +// specified. The ETag must be specified in quotes. xMsSourceIfModifiedSince is optional. A date and time value. +// Specify this header to perform the rename operation only if the source has been modified since the specified date +// and time. xMsSourceIfUnmodifiedSince is optional. A date and time value. Specify this header to perform the rename +// operation only if the source has not been modified since the specified date and time. xMsClientRequestID is a UUID +// recorded in the analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value +// in seconds. The period begins when the request is received by the service. If the timeout value elapses before the +// operation completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. +// This is required when using shared key authorization. +func (client pathClient) Create(ctx context.Context, filesystem string, pathParameter string, resource PathResourceType, continuation *string, mode PathRenameModeType, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, xMsUmask *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathCreateResponse, error) { + if err := validate([]validation{ + { targetValue: xMsLeaseID, + constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: xMsSourceLeaseID, + constraints: []constraint{ {target: "xMsSourceLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsSourceLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.createPreparer(filesystem, pathParameter, resource, continuation, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, xMsCacheControl, xMsContentType, xMsContentEncoding, xMsContentLanguage, xMsContentDisposition, xMsRenameSource, xMsLeaseID, xMsSourceLeaseID, xMsProperties, xMsPermissions, xMsUmask, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsSourceIfMatch, xMsSourceIfNoneMatch, xMsSourceIfModifiedSince, xMsSourceIfUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathCreateResponse), err +} + +// createPreparer prepares the Create request. +func (client pathClient) createPreparer(filesystem string, pathParameter string, resource PathResourceType, continuation *string, mode PathRenameModeType, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, xMsUmask *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("PUT", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if resource != PathResourceNone { + params.Set("resource", string(resource)) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if mode != PathRenameModeNone { + params.Set("mode", string(mode)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if cacheControl != nil { + req.Header.Set("Cache-Control", *cacheControl) + } + if contentEncoding != nil { + req.Header.Set("Content-Encoding", *contentEncoding) + } + if contentLanguage != nil { + req.Header.Set("Content-Language", *contentLanguage) + } + if contentDisposition != nil { + req.Header.Set("Content-Disposition", *contentDisposition) + } + if xMsCacheControl != nil { + req.Header.Set("x-ms-cache-control", *xMsCacheControl) + } + if xMsContentType != nil { + req.Header.Set("x-ms-content-type", *xMsContentType) + } + if xMsContentEncoding != nil { + req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) + } + if xMsContentLanguage != nil { + req.Header.Set("x-ms-content-language", *xMsContentLanguage) + } + if xMsContentDisposition != nil { + req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) + } + if xMsRenameSource != nil { + req.Header.Set("x-ms-rename-source", *xMsRenameSource) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsSourceLeaseID != nil { + req.Header.Set("x-ms-source-lease-id", *xMsSourceLeaseID) + } + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsPermissions != nil { + req.Header.Set("x-ms-permissions", *xMsPermissions) + } + if xMsUmask != nil { + req.Header.Set("x-ms-umask", *xMsUmask) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsSourceIfMatch != nil { + req.Header.Set("x-ms-source-if-match", *xMsSourceIfMatch) + } + if xMsSourceIfNoneMatch != nil { + req.Header.Set("x-ms-source-if-none-match", *xMsSourceIfNoneMatch) + } + if xMsSourceIfModifiedSince != nil { + req.Header.Set("x-ms-source-if-modified-since", *xMsSourceIfModifiedSince) + } + if xMsSourceIfUnmodifiedSince != nil { + req.Header.Set("x-ms-source-if-unmodified-since", *xMsSourceIfUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// createResponder handles the response to the Create request. +func (client pathClient) createResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK,http.StatusCreated) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathCreateResponse{rawResponse: resp.Response()}, err +} + +// Delete delete the file or directory. This operation supports conditional HTTP requests. For more information, see +// [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. recursive is required and +// valid only when the resource is a directory. If "true", all paths beneath the directory will be deleted. If "false" +// and the directory is non-empty, an error occurs. continuation is optional. When deleting a directory, the number of +// paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a +// continuation token is returned in this response header. When a continuation token is returned in the response, it +// must be specified in a subsequent invocation of the delete operation to continue deleting the directory. xMsLeaseID +// is the lease ID must be specified if there is an active lease. ifMatch is optional. An ETag value. Specify this +// header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified +// in quotes. ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to +// perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in +// quotes. ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. +// Specify this header to perform the operation only if the resource has not been modified since the specified date and +// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client pathClient) Delete(ctx context.Context, filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathDeleteResponse, error) { + if err := validate([]validation{ + { targetValue: xMsLeaseID, + constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.deletePreparer(filesystem, pathParameter, recursive, continuation, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathDeleteResponse), err +} + +// deletePreparer prepares the Delete request. +func (client pathClient) deletePreparer(filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("DELETE", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if recursive != nil { + params.Set("recursive", strconv.FormatBool(*recursive)) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// deleteResponder handles the response to the Delete request. +func (client pathClient) deleteResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathDeleteResponse{rawResponse: resp.Response()}, err +} + +// GetProperties get Properties returns all system and user defined properties for a path. Get Status returns all +// system defined properties for a path. Get Access Control List returns the access control list for a path. This +// operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob +// Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. action is optional. If the +// value is "getStatus" only the system defined properties for the path are returned. If the value is +// "getAccessControl" the access control list is returned in the response headers (Hierarchical Namespace must be +// enabled for the account), otherwise the properties are returned. upn is optional. Valid only when Hierarchical +// Namespace is enabled for the account. If "true", the user identity values returned in the x-ms-owner, x-ms-group, +// and x-ms-acl response headers will be transformed from Azure Active Directory Object IDs to User Principal Names. +// If "false", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that +// group and application Object IDs are not translated because they do not have unique friendly names. xMsLeaseID is +// optional. If this header is specified, the operation will be performed only if both of the following conditions are +// met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path. +// ifMatch is optional. An ETag value. Specify this header to perform the operation only if the resource's ETag +// matches the value specified. The ETag must be specified in quotes. ifNoneMatch is optional. An ETag value or the +// special wildcard ("*") value. Specify this header to perform the operation only if the resource's ETag does not +// match the value specified. The ETag must be specified in quotes. ifModifiedSince is optional. A date and time value. +// Specify this header to perform the operation only if the resource has been modified since the specified date and +// time. ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has not been modified since the specified date and time. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client pathClient) GetProperties(ctx context.Context, filesystem string, pathParameter string, action PathGetPropertiesActionType, upn *bool, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathGetPropertiesResponse, error) { + if err := validate([]validation{ + { targetValue: xMsLeaseID, + constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.getPropertiesPreparer(filesystem, pathParameter, action, upn, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPropertiesResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathGetPropertiesResponse), err +} + +// getPropertiesPreparer prepares the GetProperties request. +func (client pathClient) getPropertiesPreparer(filesystem string, pathParameter string, action PathGetPropertiesActionType, upn *bool, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("HEAD", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if action != PathGetPropertiesActionNone { + params.Set("action", string(action)) + } + if upn != nil { + params.Set("upn", strconv.FormatBool(*upn)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// getPropertiesResponder handles the response to the GetProperties request. +func (client pathClient) getPropertiesResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathGetPropertiesResponse{rawResponse: resp.Response()}, err +} + +// Lease create and manage a lease to restrict write and delete access to the path. This operation supports conditional +// HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// xMsLeaseAction is there are five lease actions: "acquire", "break", "change", "renew", and "release". Use "acquire" +// and specify the "x-ms-proposed-lease-id" and "x-ms-lease-duration" to acquire a new lease. Use "break" to break an +// existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease +// operation except break and release can be performed on the file. When a lease is successfully broken, the response +// indicates the interval in seconds until a new lease can be acquired. Use "change" and specify the current lease ID +// in "x-ms-lease-id" and the new lease ID in "x-ms-proposed-lease-id" to change the lease ID of an active lease. Use +// "renew" and specify the "x-ms-lease-id" to renew an existing lease. Use "release" and specify the "x-ms-lease-id" to +// release a lease. filesystem is the filesystem identifier. pathParameter is the file or directory path. +// xMsLeaseDuration is the lease duration is required to acquire a lease, and specifies the duration of the lease in +// seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. xMsLeaseBreakPeriod is the +// lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. +// The lease break duration must be between 0 and 60 seconds. xMsLeaseID is required when "x-ms-lease-action" is +// "renew", "change" or "release". For the renew and release actions, this must match the current lease ID. +// xMsProposedLeaseID is required when "x-ms-lease-action" is "acquire" or "change". A lease will be acquired with +// this lease ID if the operation is successful. ifMatch is optional. An ETag value. Specify this header to perform +// the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes. +// ifNoneMatch is optional. An ETag value or the special wildcard ("*") value. Specify this header to perform the +// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. +// ifModifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has been modified since the specified date and time. ifUnmodifiedSince is optional. A date and time value. +// Specify this header to perform the operation only if the resource has not been modified since the specified date and +// time. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client pathClient) Lease(ctx context.Context, xMsLeaseAction PathLeaseActionType, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathLeaseResponse, error) { + if err := validate([]validation{ + { targetValue: xMsLeaseID, + constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: xMsProposedLeaseID, + constraints: []constraint{ {target: "xMsProposedLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.leasePreparer(xMsLeaseAction, filesystem, pathParameter, xMsLeaseDuration, xMsLeaseBreakPeriod, xMsLeaseID, xMsProposedLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.leaseResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathLeaseResponse), err +} + +// leasePreparer prepares the Lease request. +func (client pathClient) leasePreparer(xMsLeaseAction PathLeaseActionType, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("POST", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + req.Header.Set("x-ms-lease-action", string(xMsLeaseAction)) + if xMsLeaseDuration != nil { + req.Header.Set("x-ms-lease-duration", strconv.FormatInt(int64(*xMsLeaseDuration), 10)) + } + if xMsLeaseBreakPeriod != nil { + req.Header.Set("x-ms-lease-break-period", strconv.FormatInt(int64(*xMsLeaseBreakPeriod), 10)) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsProposedLeaseID != nil { + req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// leaseResponder handles the response to the Lease request. +func (client pathClient) leaseResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK,http.StatusCreated,http.StatusAccepted) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathLeaseResponse{rawResponse: resp.Response()}, err +} + +// List list filesystem paths and their properties. +// +// recursive is if "true", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If +// "directory" is specified, the list will only include paths that share the same root. filesystem is the filesystem +// identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the +// dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have +// between 3 and 63 characters. directory is filters results to paths within the specified directory. An error occurs +// if the directory does not exist. continuation is the number of paths returned with each invocation is limited. If +// the number of paths to be returned exceeds this limit, a continuation token is returned in the response header +// x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent +// invocation of the list operation to continue listing the paths. maxResults is an optional value that specifies the +// maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items. +// upn is optional. Valid only when Hierarchical Namespace is enabled for the account. If "true", the user identity +// values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory +// Object IDs to User Principal Names. If "false", the values will be returned as Azure Active Directory Object IDs. +// The default value is false. Note that group and application Object IDs are not translated because they do not have +// unique friendly names. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and +// correlation. timeout is an optional operation timeout value in seconds. The period begins when the request is +// received by the service. If the timeout value elapses before the operation completes, the operation fails. xMsDate +// is specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key +// authorization. +func (client pathClient) List(ctx context.Context, recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathList, error) { + if err := validate([]validation{ + { targetValue: maxResults, + constraints: []constraint{ {target: "maxResults", name: null, rule: false , + chain: []constraint{ {target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}, + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.listPreparer(recursive, filesystem, directory, continuation, maxResults, upn, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathList), err +} + +// listPreparer prepares the List request. +func (client pathClient) listPreparer(recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("GET", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if directory != nil && len(*directory) > 0 { + params.Set("directory", *directory) + } + params.Set("recursive", strconv.FormatBool(recursive)) + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if maxResults != nil { + params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) + } + if upn != nil { + params.Set("upn", strconv.FormatBool(*upn)) + } + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// listResponder handles the response to the List request. +func (client pathClient) listResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + result:= &PathList{rawResponse: resp.Response()} + if err != nil { + return result, err + } + defer resp.Response().Body.Close() + b, err:= ioutil.ReadAll(resp.Response().Body) + if err != nil { + return result, err + } + if len(b) > 0 { + b = removeBOM(b) + err = json.Unmarshal(b, result) + if err != nil { + return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") + } + } + return result, nil +} + +// Read read the contents of a file. For read operations, range requests are supported. This operation supports +// conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// filesystem is the filesystem identifier. pathParameter is the file or directory path. rangeParameter is the HTTP +// Range request header specifies one or more byte ranges of the resource to be retrieved. xMsLeaseID is optional. If +// this header is specified, the operation will be performed only if both of the following conditions are met: i) the +// path's lease is currently active and ii) the lease ID specified in the request matches that of the path. ifMatch is +// optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value +// specified. The ETag must be specified in quotes. ifNoneMatch is optional. An ETag value or the special wildcard +// ("*") value. Specify this header to perform the operation only if the resource's ETag does not match the value +// specified. The ETag must be specified in quotes. ifModifiedSince is optional. A date and time value. Specify this +// header to perform the operation only if the resource has been modified since the specified date and time. +// ifUnmodifiedSince is optional. A date and time value. Specify this header to perform the operation only if the +// resource has not been modified since the specified date and time. xMsClientRequestID is a UUID recorded in the +// analytics logs for troubleshooting and correlation. timeout is an optional operation timeout value in seconds. The +// period begins when the request is received by the service. If the timeout value elapses before the operation +// completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is +// required when using shared key authorization. +func (client pathClient) Read(ctx context.Context, filesystem string, pathParameter string, rangeParameter *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ReadResponse, error) { + if err := validate([]validation{ + { targetValue: xMsLeaseID, + constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.readPreparer(filesystem, pathParameter, rangeParameter, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.readResponder}, req) + if err != nil { + return nil, err + } + return resp.(*ReadResponse), err +} + +// readPreparer prepares the Read request. +func (client pathClient) readPreparer(filesystem string, pathParameter string, rangeParameter *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("GET", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if rangeParameter != nil { + req.Header.Set("Range", *rangeParameter) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// readResponder handles the response to the Read request. +func (client pathClient) readResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK,http.StatusPartialContent) + if resp == nil { + return nil, err + } + return &ReadResponse{rawResponse: resp.Response()}, err +} + +// Update uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties +// for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This +// operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob +// Service +// Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). +// +// action is the action must be "append" to upload data to be appended to a file, "flush" to flush previously uploaded +// data to a file, "setProperties" to set the properties of a file or directory, or "setAccessControl" to set the +// owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be +// enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes +// permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are +// mutually exclusive. filesystem is the filesystem identifier. pathParameter is the file or directory path. position +// is this parameter allows the caller to upload data in parallel and control the order in which it is appended to the +// file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to +// the file. The value must be the position where the data is to be appended. Uploaded data is not immediately +// flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter +// must be specified and equal to the length of the file after all data has been written, and there must not be a +// request entity body included with the request. retainUncommittedData is valid only for flush operations. If "true", +// uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after +// the flush operation. The default is false. Data at offsets less than the specified position are written to the +// file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a +// future flush operation. closeParameter is azure Storage Events allow applications to receive notifications when +// files change. When Azure Storage Events are enabled, a file changed event is raised. This event has a property +// indicating whether this is the final change to distinguish the difference between an intermediate flush to a file +// stream and the final close of a file stream. The close query parameter is valid only when the action is "flush" and +// change notifications are enabled. If the value of close is "true" and the flush operation completes successfully, +// the service raises a file change notification with a property indicating that this is the final update (the file +// stream has been closed). If "false" a change notification is raised indicating the file has changed. The default is +// false. This query parameter is set to true by the Hadoop ABFS driver to indicate that the file stream has been +// closed." contentLength is required for "Append Data" and "Flush Data". Must be 0 for "Flush Data". Must be the +// length of the request content in bytes for "Append Data". xMsLeaseID is the lease ID must be specified if there is +// an active lease. xMsCacheControl is optional and only valid for flush and set properties operations. The service +// stores this value and includes it in the "Cache-Control" response header for "Read File" operations. xMsContentType +// is optional and only valid for flush and set properties operations. The service stores this value and includes it +// in the "Content-Type" response header for "Read File" operations. xMsContentDisposition is optional and only valid +// for flush and set properties operations. The service stores this value and includes it in the "Content-Disposition" +// response header for "Read File" operations. xMsContentEncoding is optional and only valid for flush and set +// properties operations. The service stores this value and includes it in the "Content-Encoding" response header for +// "Read File" operations. xMsContentLanguage is optional and only valid for flush and set properties operations. The +// service stores this value and includes it in the "Content-Language" response header for "Read File" operations. +// xMsContentMd5 is optional and only valid for "Flush & Set Properties" operations. The service stores this value and +// includes it in the "Content-Md5" response header for "Read & Get Properties" operations. If this property is not +// specified on the request, then the property will be cleared for the file. Subsequent calls to "Read & Get +// Properties" will not return this property unless it is explicitly set on that file again. xMsProperties is optional. +// User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and +// value pairs "n1=v1, n2=v2, ...", where each value is a base64 encoded string. Note that the string may only contain +// ASCII characters in the ISO-8859-1 character set. Valid only for the setProperties operation. If the file or +// directory exists, any properties not included in the list will be removed. All properties are removed if the header +// is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then +// make a conditional request with the E-Tag and include values for all properties. xMsOwner is optional and valid only +// for the setAccessControl operation. Sets the owner of the file or directory. xMsGroup is optional and valid only for +// the setAccessControl operation. Sets the owning group of the file or directory. xMsPermissions is optional and only +// valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the +// file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also +// supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction +// with x-ms-acl. xMsACL is optional and valid only for the setAccessControl operation. Sets POSIX access control +// rights on files and directories. The value is a comma-separated list of access control entries that fully replaces +// the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or +// group identifier, and permissions in the format "[scope:][type]:[id]:[permissions]". The scope must be "default" to +// indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the +// access ACL. There are four ACE types: "user" grants rights to the owner or a named user, "group" grants rights to +// the owning group or a named group, "mask" restricts rights granted to named users and the members of groups, and +// "other" grants rights to all users not found in any of the other entries. The user or group identifier is omitted +// for entries of type "mask" and "other". The user or group identifier is also omitted for the owner and owning +// group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the +// second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If +// access is not granted, the '-' character is used to denote that the permission is denied. For example, the following +// ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning +// group, and nothing to everyone else: "user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx". Invalid +// in conjunction with x-ms-permissions. ifMatch is optional for Flush Data and Set Properties, but invalid for Append +// Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value +// specified. The ETag must be specified in quotes. ifNoneMatch is optional for Flush Data and Set Properties, but +// invalid for Append Data. An ETag value or the special wildcard ("*") value. Specify this header to perform the +// operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes. +// ifModifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. +// Specify this header to perform the operation only if the resource has been modified since the specified date and +// time. ifUnmodifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time +// value. Specify this header to perform the operation only if the resource has not been modified since the specified +// date and time. requestBody is valid only for append operations. The data to be uploaded and appended to the file. +// requestBody will be closed upon successful return. Callers should ensure closure when receiving an +// error.xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an +// optional operation timeout value in seconds. The period begins when the request is received by the service. If the +// timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated +// Universal Time (UTC) for the request. This is required when using shared key authorization. +func (client pathClient) Update(ctx context.Context, action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathUpdateResponse, error) { + if err := validate([]validation{ + { targetValue: contentLength, + constraints: []constraint{ {target: "contentLength", name: null, rule: false , + chain: []constraint{ {target: "contentLength", name: inclusiveMinimum, rule: 0, chain: nil }, + }}}}, + { targetValue: xMsLeaseID, + constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , + chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: filesystem, + constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, + {target: "filesystem", name: minLength, rule: 3, chain: nil }, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + { targetValue: xMsClientRequestID, + constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , + chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, + }}}}, + { targetValue: timeout, + constraints: []constraint{ {target: "timeout", name: null, rule: false , + chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, + }}}}}); err != nil { + return nil, err + } + req, err := client.updatePreparer(action, filesystem, pathParameter, position, retainUncommittedData, closeParameter, contentLength, xMsLeaseID, xMsCacheControl, xMsContentType, xMsContentDisposition, xMsContentEncoding, xMsContentLanguage, xMsContentMd5, xMsProperties, xMsOwner, xMsGroup, xMsPermissions, xMsACL, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, body, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.updateResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathUpdateResponse), err +} + +// updatePreparer prepares the Update request. +func (client pathClient) updatePreparer(action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("PATCH", client.url, body) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + params.Set("action", string(action)) + if position != nil { + params.Set("position", strconv.FormatInt(*position, 10)) + } + if retainUncommittedData != nil { + params.Set("retainUncommittedData", strconv.FormatBool(*retainUncommittedData)) + } + if closeParameter != nil { + params.Set("close", strconv.FormatBool(*closeParameter)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if contentLength != nil { + req.Header.Set("Content-Length", strconv.FormatInt(*contentLength, 10)) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsCacheControl != nil { + req.Header.Set("x-ms-cache-control", *xMsCacheControl) + } + if xMsContentType != nil { + req.Header.Set("x-ms-content-type", *xMsContentType) + } + if xMsContentDisposition != nil { + req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) + } + if xMsContentEncoding != nil { + req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) + } + if xMsContentLanguage != nil { + req.Header.Set("x-ms-content-language", *xMsContentLanguage) + } + if xMsContentMd5 != nil { + req.Header.Set("x-ms-content-md5", *xMsContentMd5) + } + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsOwner != nil { + req.Header.Set("x-ms-owner", *xMsOwner) + } + if xMsGroup != nil { + req.Header.Set("x-ms-group", *xMsGroup) + } + if xMsPermissions != nil { + req.Header.Set("x-ms-permissions", *xMsPermissions) + } + if xMsACL != nil { + req.Header.Set("x-ms-acl", *xMsACL) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + if xMsVersion != nil { + req.Header.Set("x-ms-version", *client.XMsVersion) + } + return req, nil +} + +// updateResponder handles the response to the Update request. +func (client pathClient) updateResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK,http.StatusAccepted) + if resp == nil { + return nil, err + } + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathUpdateResponse{rawResponse: resp.Response()}, err +} + diff --git a/azbfs/zz_generated_responder_policy.go b/azbfs/zz_generated_responder_policy.go index b1d535484..9c35c7723 100644 --- a/azbfs/zz_generated_responder_policy.go +++ b/azbfs/zz_generated_responder_policy.go @@ -55,7 +55,7 @@ func validateResponse(resp pipeline.Response, successStatusCodes ...int) error { defer resp.Response().Body.Close() b, err := ioutil.ReadAll(resp.Response().Body) if err != nil { - return NewResponseError(err, resp.Response(), "failed to read response body") + return err } // the service code, description and details will be populated during unmarshalling responseError := NewResponseError(nil, resp.Response(), resp.Response().Status) diff --git a/azbfs/zz_generated_version.go b/azbfs/zz_generated_version.go index b8072a456..6fe430b24 100644 --- a/azbfs/zz_generated_version.go +++ b/azbfs/zz_generated_version.go @@ -5,7 +5,7 @@ package azbfs // UserAgent returns the UserAgent string to use when sending http.Requests. func UserAgent() string { - return "Azure-SDK-For-Go/0.0.0 azbfs/2018-06-17" + return "Azure-SDK-For-Go/0.0.0 azbfs/2018-11-09" } // Version returns the semantic version (see http://semver.org) of the client. From d0a75b12abd09bd8e18c23bf425fba4ee43c664c Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 15:07:00 +1300 Subject: [PATCH 18/64] Re-work our wrapper classes to match the new generated code --- azbfs/url_directory.go | 18 ++-- azbfs/url_file.go | 64 +++++++------- azbfs/url_filesystem.go | 18 ++-- azbfs/zz_response_model.go | 124 ++++++++++++++-------------- cmd/copyDownloadBlobFSEnumerator.go | 16 +--- cmd/copyEnumeratorHelper.go | 4 +- 6 files changed, 120 insertions(+), 124 deletions(-) diff --git a/azbfs/url_directory.go b/azbfs/url_directory.go index b7573d971..3ccf34b2b 100644 --- a/azbfs/url_directory.go +++ b/azbfs/url_directory.go @@ -11,7 +11,7 @@ var directoryResourceName = "directory" // constant value for the resource query // A DirectoryURL represents a URL to the Azure Storage directory allowing you to manipulate its directories and files. type DirectoryURL struct { - directoryClient managementClient + directoryClient pathClient // filesystem is the filesystem identifier filesystem string // pathParameter is the file or directory path @@ -24,7 +24,7 @@ func NewDirectoryURL(url url.URL, p pipeline.Pipeline) DirectoryURL { panic("p can't be nil") } urlParts := NewBfsURLParts(url) - directoryClient := newManagementClient(url, p) + directoryClient := newPathClient(url, p) return DirectoryURL{directoryClient: directoryClient, filesystem: urlParts.FileSystemName, pathParameter: urlParts.DirectoryOrFilePath} } @@ -65,8 +65,8 @@ func (d DirectoryURL) NewDirectoryURL(dirName string) DirectoryURL { // Create creates a new directory within a File System func (d DirectoryURL) Create(ctx context.Context) (*DirectoryCreateResponse, error) { - resp, err := d.directoryClient.CreatePath(ctx, d.filesystem, d.pathParameter, &directoryResourceName, nil, - nil, nil, nil, nil, nil, nil, + resp, err := d.directoryClient.Create(ctx, d.filesystem, d.pathParameter, PathResourceDirectory, nil, + PathRenameModeNone, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, @@ -77,15 +77,15 @@ func (d DirectoryURL) Create(ctx context.Context) (*DirectoryCreateResponse, err // Delete removes the specified empty directory. Note that the directory must be empty before it can be deleted.. // For more information, see https://docs.microsoft.com/rest/api/storageservices/delete-directory. func (d DirectoryURL) Delete(ctx context.Context, continuationString *string, recursive bool) (*DirectoryDeleteResponse, error) { - resp, err := d.directoryClient.DeletePath(ctx, d.filesystem, d.pathParameter, &recursive, continuationString, nil, + resp, err := d.directoryClient.Delete(ctx, d.filesystem, d.pathParameter, &recursive, continuationString, nil, nil, nil, nil, nil, nil, nil, nil) return (*DirectoryDeleteResponse)(resp), err } // GetProperties returns the directory's metadata and system properties. func (d DirectoryURL) GetProperties(ctx context.Context) (*DirectoryGetPropertiesResponse, error) { - resp, err := d.directoryClient.GetPathProperties(ctx, d.filesystem, d.pathParameter, nil, nil, nil, - nil, nil, nil, nil, nil) + resp, err := d.directoryClient.GetProperties(ctx, d.filesystem, d.pathParameter, PathGetPropertiesActionGetStatus, nil, nil, + nil, nil, nil, nil, nil, nil, nil) return (*DirectoryGetPropertiesResponse)(resp), err } @@ -110,8 +110,8 @@ func (d DirectoryURL) ListDirectorySegment(ctx context.Context, marker *string, // and listPath for filesystem with directory path set in the path parameter var maxEntriesInListOperation = int32(1000) - resp, err := d.FileSystemURL().fileSystemClient.ListPaths(ctx, recursive, d.filesystem, fileSystemResourceName, &d.pathParameter, marker, - &maxEntriesInListOperation, nil, nil, nil) + resp, err := d.directoryClient.List(ctx, recursive, d.filesystem, &d.pathParameter, marker, + &maxEntriesInListOperation, nil, nil, nil, nil) return (*DirectoryListResponse)(resp), err } diff --git a/azbfs/url_file.go b/azbfs/url_file.go index ffe6c4ce3..916d08666 100644 --- a/azbfs/url_file.go +++ b/azbfs/url_file.go @@ -7,12 +7,11 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" "io" "net/http" - "strconv" ) // A FileURL represents a URL to an Azure Storage file. type FileURL struct { - fileClient managementClient + fileClient pathClient fileSystemName string path string } @@ -22,7 +21,7 @@ func NewFileURL(url url.URL, p pipeline.Pipeline) FileURL { if p == nil { panic("p can't be nil") } - fileClient := newManagementClient(url, p) + fileClient := newPathClient(url, p) urlParts := NewBfsURLParts(url) return FileURL{fileClient: fileClient, fileSystemName: urlParts.FileSystemName, path: urlParts.DirectoryOrFilePath} @@ -46,10 +45,9 @@ func (f FileURL) WithPipeline(p pipeline.Pipeline) FileURL { // Create creates a new file or replaces a file. Note that this method only initializes the file. // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/create-file. -func (f FileURL) Create(ctx context.Context) (*CreatePathResponse, error) { - fileType := "file" - return f.fileClient.CreatePath(ctx, f.fileSystemName, f.path, &fileType, - nil, nil, nil, nil, nil, nil, +func (f FileURL) Create(ctx context.Context) (*PathCreateResponse, error) { + return f.fileClient.Create(ctx, f.fileSystemName, f.path, PathResourceFile, + nil, PathRenameModeNone, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, @@ -62,8 +60,8 @@ func (f FileURL) Create(ctx context.Context) (*CreatePathResponse, error) { // response header/property if the range is <= 4MB; the HTTP request fails with 400 (Bad Request) if the requested range is greater than 4MB. // For more information, see https://docs.microsoft.com/rest/api/storageservices/get-file. func (f FileURL) Download(ctx context.Context, offset int64, count int64) (*DownloadResponse, error) { - dr, err := f.fileClient.ReadPath(ctx, f.fileSystemName, f.path, (&httpRange{offset: offset, count: count}).pointers(), - nil, nil, nil, nil, nil, nil, nil) + dr, err := f.fileClient.Read(ctx, f.fileSystemName, f.path, (&httpRange{offset: offset, count: count}).pointers(), + nil, nil, nil, nil, nil, nil, nil, nil) if err != nil { return nil, err } @@ -72,7 +70,9 @@ func (f FileURL) Download(ctx context.Context, offset int64, count int64) (*Down f: f, dr: dr, ctx: ctx, - info: HTTPGetterInfo{Offset: offset, Count: count, ETag: dr.ETag()}, // TODO: Note conditional header is not currently supported in Azure File. + info: HTTPGetterInfo{Offset: offset, Count: count, ETag: dr.ETag()}, + // TODO: Note conditional header is not currently supported in Azure File. + // TODO: review the above todo, since as of 8 Feb 2019 we are on a newer version of the API }, err } @@ -99,24 +99,24 @@ func (dr *DownloadResponse) Body(o RetryReaderOptions) io.ReadCloser { // Delete immediately removes the file from the storage account. // For more information, see https://docs.microsoft.com/en-us/rest/api/storageservices/delete-file2. -func (f FileURL) Delete(ctx context.Context) (*DeletePathResponse, error) { +func (f FileURL) Delete(ctx context.Context) (*PathDeleteResponse, error) { recursive := false - return f.fileClient.DeletePath(ctx, f.fileSystemName, f.path, &recursive, + return f.fileClient.Delete(ctx, f.fileSystemName, f.path, &recursive, nil, nil, nil, nil, nil, nil, nil, nil, nil) } // GetProperties returns the file's metadata and properties. // For more information, see https://docs.microsoft.com/rest/api/storageservices/get-file-properties. -func (f FileURL) GetProperties(ctx context.Context) (*GetPathPropertiesResponse, error) { - return f.fileClient.GetPathProperties(ctx, f.fileSystemName, f.path, nil, nil, +func (f FileURL) GetProperties(ctx context.Context) (*PathGetPropertiesResponse, error) { + return f.fileClient.GetProperties(ctx, f.fileSystemName, f.path, PathGetPropertiesActionGetStatus, nil, nil, nil, nil, - nil, nil, nil) + nil, nil, nil, nil, nil) } // UploadRange writes bytes to a file. // offset indiciates the offset at which to begin writing, in bytes. -func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeker) (*UpdatePathResponse, error) { +func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeker) (*PathUpdateResponse, error) { if offset < 0 { panic("offset must be >= 0") } @@ -128,21 +128,22 @@ func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeke if count == 0 { panic("body must contain readable data whose size is > 0") } - countAsStr := strconv.FormatInt(count, 10) - // TODO the go http client has a problem with PATCH and content-length header - // TODO we should investigate and report the issue - overrideHttpVerb := "PATCH" + // TODO: does this AppendData function even work now? We use to override the Http verb, but the new API doesn't + // let us do that + // Old_TODO_text: the go http client has a problem with PATCH and content-length header + // we should investigate and report the issue + // overrideHttpVerb := "PATCH" // TransactionalContentMD5 isn't supported currently. - return f.fileClient.UpdatePath(ctx, "append", f.fileSystemName, f.path, &offset, - nil, &countAsStr, nil, nil, nil, nil, + return f.fileClient.Update(ctx, PathUpdateActionAppend, f.fileSystemName, f.path, &offset, + nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, &overrideHttpVerb, body, nil, nil, nil) + nil, nil, nil, nil, nil, nil, body, nil, nil, nil) } // flushes writes previously uploaded data to a file -func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*UpdatePathResponse, error) { +func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*PathUpdateResponse, error) { if fileSize < 0 { panic("fileSize must be >= 0") } @@ -151,14 +152,19 @@ func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*UpdatePathResp // azcopy does not need this retainUncommittedData := false - // TODO the go http client has a problem with PATCH and content-length header - // TODO we should investigate and report the issue - overrideHttpVerb := "PATCH" + // TODO: does this FlushData function even work now? We use to override the Http verb, but the new API doesn't + // let us do that + // Old_TODO_text: the go http client has a problem with PATCH and content-length header + // we should investigate and report the issue + // overrideHttpVerb := "PATCH" + + // TODO: feb 2019 API update: review the use of closeParameter here. Should it be true? + // Doc implies only make it true if // TransactionalContentMD5 isn't supported currently. - return f.fileClient.UpdatePath(ctx, "flush", f.fileSystemName, f.path, &fileSize, + return f.fileClient.Update(ctx, PathUpdateActionFlush, f.fileSystemName, f.path, &fileSize, &retainUncommittedData, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - &overrideHttpVerb, nil, nil, nil, nil) + nil, nil, nil, nil, nil) } diff --git a/azbfs/url_filesystem.go b/azbfs/url_filesystem.go index 929978bbe..eb3d87e9a 100644 --- a/azbfs/url_filesystem.go +++ b/azbfs/url_filesystem.go @@ -7,11 +7,9 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" ) -const fileSystemResourceName = "filesystem" // constant value for the resource query parameter - // A FileSystemURL represents a URL to the Azure Storage Blob File System allowing you to manipulate its directories and files. type FileSystemURL struct { - fileSystemClient managementClient + fileSystemClient filesystemClient name string } @@ -20,7 +18,7 @@ func NewFileSystemURL(url url.URL, p pipeline.Pipeline) FileSystemURL { if p == nil { panic("p can't be nil") } - fileSystemClient := newManagementClient(url, p) + fileSystemClient := newFilesystemClient(url, p) urlParts := NewBfsURLParts(url) return FileSystemURL{fileSystemClient: fileSystemClient, name: urlParts.FileSystemName} @@ -62,17 +60,17 @@ func (s FileSystemURL) NewRootDirectoryURL() DirectoryURL { // Create creates a new file system within a storage account. If a file system with the same name already exists, the operation fails. // quotaInGB specifies the maximum size of the file system in gigabytes, 0 means you accept service's default quota. -func (s FileSystemURL) Create(ctx context.Context) (*CreateFilesystemResponse, error) { - return s.fileSystemClient.CreateFilesystem(ctx, s.name, fileSystemResourceName, nil, nil, nil, nil) +func (s FileSystemURL) Create(ctx context.Context) (*FilesystemCreateResponse, error) { + return s.fileSystemClient.Create(ctx, s.name, nil, nil, nil, nil) } // Delete marks the specified file system for deletion. // The file system and any files contained within it are later deleted during garbage collection. -func (s FileSystemURL) Delete(ctx context.Context) (*DeleteFilesystemResponse, error) { - return s.fileSystemClient.DeleteFilesystem(ctx, s.name, fileSystemResourceName, nil, nil, nil, nil, nil) +func (s FileSystemURL) Delete(ctx context.Context) (*FilesystemDeleteResponse, error) { + return s.fileSystemClient.Delete(ctx, s.name, nil, nil, nil, nil, nil) } // GetProperties returns all user-defined metadata and system properties for the specified file system or file system snapshot. -func (s FileSystemURL) GetProperties(ctx context.Context) (*GetFilesystemPropertiesResponse, error) { - return s.fileSystemClient.GetFilesystemProperties(ctx, s.name, fileSystemResourceName, nil, nil, nil) +func (s FileSystemURL) GetProperties(ctx context.Context) (*FilesystemGetPropertiesResponse, error) { + return s.fileSystemClient.GetProperties(ctx, s.name, nil, nil, nil) } diff --git a/azbfs/zz_response_model.go b/azbfs/zz_response_model.go index 2c2040095..ea9c5281c 100644 --- a/azbfs/zz_response_model.go +++ b/azbfs/zz_response_model.go @@ -8,262 +8,262 @@ import ( // DirectoryCreateResponse is the CreatePathResponse response type returned for directory specific operations // The type is used to establish difference in the response for file and directory operations since both type of // operations has same response type. -type DirectoryCreateResponse CreatePathResponse +type DirectoryCreateResponse PathCreateResponse // Response returns the raw HTTP response object. func (dcr DirectoryCreateResponse) Response() *http.Response { - return CreatePathResponse(dcr).Response() + return PathCreateResponse(dcr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (dcr DirectoryCreateResponse) StatusCode() int { - return CreatePathResponse(dcr).StatusCode() + return PathCreateResponse(dcr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (dcr DirectoryCreateResponse) Status() string { - return CreatePathResponse(dcr).Status() + return PathCreateResponse(dcr).Status() } // ContentLength returns the value for header Content-Length. -func (dcr DirectoryCreateResponse) ContentLength() string { - return CreatePathResponse(dcr).ContentLength() +func (dcr DirectoryCreateResponse) ContentLength() int64 { + return PathCreateResponse(dcr).ContentLength() } // Date returns the value for header Date. func (dcr DirectoryCreateResponse) Date() string { - return CreatePathResponse(dcr).Date() + return PathCreateResponse(dcr).Date() } // ETag returns the value for header ETag. func (dcr DirectoryCreateResponse) ETag() string { - return CreatePathResponse(dcr).ETag() + return PathCreateResponse(dcr).ETag() } // LastModified returns the value for header Last-Modified. func (dcr DirectoryCreateResponse) LastModified() string { - return CreatePathResponse(dcr).LastModified() + return PathCreateResponse(dcr).LastModified() } // XMsContinuation returns the value for header x-ms-continuation. func (dcr DirectoryCreateResponse) XMsContinuation() string { - return CreatePathResponse(dcr).XMsContinuation() + return PathCreateResponse(dcr).XMsContinuation() } // XMsRequestID returns the value for header x-ms-request-id. func (dcr DirectoryCreateResponse) XMsRequestID() string { - return CreatePathResponse(dcr).XMsRequestID() + return PathCreateResponse(dcr).XMsRequestID() } // XMsVersion returns the value for header x-ms-version. func (dcr DirectoryCreateResponse) XMsVersion() string { - return CreatePathResponse(dcr).XMsVersion() + return PathCreateResponse(dcr).XMsVersion() } // DirectoryDeleteResponse is the DeletePathResponse response type returned for directory specific operations // The type is used to establish difference in the response for file and directory operations since both type of // operations has same response type. -type DirectoryDeleteResponse DeletePathResponse +type DirectoryDeleteResponse PathDeleteResponse // Response returns the raw HTTP response object. func (ddr DirectoryDeleteResponse) Response() *http.Response { - return DeletePathResponse(ddr).Response() + return PathDeleteResponse(ddr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (ddr DirectoryDeleteResponse) StatusCode() int { - return DeletePathResponse(ddr).StatusCode() + return PathDeleteResponse(ddr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (ddr DirectoryDeleteResponse) Status() string { - return DeletePathResponse(ddr).Status() + return PathDeleteResponse(ddr).Status() } // Date returns the value for header Date. func (ddr DirectoryDeleteResponse) Date() string { - return DeletePathResponse(ddr).Date() + return PathDeleteResponse(ddr).Date() } // XMsContinuation returns the value for header x-ms-continuation. func (ddr DirectoryDeleteResponse) XMsContinuation() string { - return DeletePathResponse(ddr).XMsContinuation() + return PathDeleteResponse(ddr).XMsContinuation() } // XMsRequestID returns the value for header x-ms-request-id. func (ddr DirectoryDeleteResponse) XMsRequestID() string { - return DeletePathResponse(ddr).XMsRequestID() + return PathDeleteResponse(ddr).XMsRequestID() } // XMsVersion returns the value for header x-ms-version. func (ddr DirectoryDeleteResponse) XMsVersion() string { - return DeletePathResponse(ddr).XMsVersion() + return PathDeleteResponse(ddr).XMsVersion() } // DirectoryGetPropertiesResponse is the GetPathPropertiesResponse response type returned for directory specific operations // The type is used to establish difference in the response for file and directory operations since both type of // operations has same response type. -type DirectoryGetPropertiesResponse GetPathPropertiesResponse +type DirectoryGetPropertiesResponse PathGetPropertiesResponse // Response returns the raw HTTP response object. func (dgpr DirectoryGetPropertiesResponse) Response() *http.Response { - return GetPathPropertiesResponse(dgpr).Response() + return PathGetPropertiesResponse(dgpr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (dgpr DirectoryGetPropertiesResponse) StatusCode() int { - return GetPathPropertiesResponse(dgpr).StatusCode() + return PathGetPropertiesResponse(dgpr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (dgpr DirectoryGetPropertiesResponse) Status() string { - return GetPathPropertiesResponse(dgpr).Status() + return PathGetPropertiesResponse(dgpr).Status() } // AcceptRanges returns the value for header Accept-Ranges. func (dgpr DirectoryGetPropertiesResponse) AcceptRanges() string { - return GetPathPropertiesResponse(dgpr).AcceptRanges() + return PathGetPropertiesResponse(dgpr).AcceptRanges() } // CacheControl returns the value for header Cache-Control. func (dgpr DirectoryGetPropertiesResponse) CacheControl() string { - return GetPathPropertiesResponse(dgpr).CacheControl() + return PathGetPropertiesResponse(dgpr).CacheControl() } // ContentDisposition returns the value for header Content-Disposition. func (dgpr DirectoryGetPropertiesResponse) ContentDisposition() string { - return GetPathPropertiesResponse(dgpr).ContentDisposition() + return PathGetPropertiesResponse(dgpr).ContentDisposition() } // ContentEncoding returns the value for header Content-Encoding. func (dgpr DirectoryGetPropertiesResponse) ContentEncoding() string { - return GetPathPropertiesResponse(dgpr).ContentEncoding() + return PathGetPropertiesResponse(dgpr).ContentEncoding() } // ContentLanguage returns the value for header Content-Language. func (dgpr DirectoryGetPropertiesResponse) ContentLanguage() string { - return GetPathPropertiesResponse(dgpr).ContentLanguage() + return PathGetPropertiesResponse(dgpr).ContentLanguage() } // ContentLength returns the value for header Content-Length. -func (dgpr DirectoryGetPropertiesResponse) ContentLength() string { - return GetPathPropertiesResponse(dgpr).ContentLength() +func (dgpr DirectoryGetPropertiesResponse) ContentLength() int64 { + return PathGetPropertiesResponse(dgpr).ContentLength() } // ContentRange returns the value for header Content-Range. func (dgpr DirectoryGetPropertiesResponse) ContentRange() string { - return GetPathPropertiesResponse(dgpr).ContentRange() + return PathGetPropertiesResponse(dgpr).ContentRange() } // ContentType returns the value for header Content-Type. func (dgpr DirectoryGetPropertiesResponse) ContentType() string { - return GetPathPropertiesResponse(dgpr).ContentType() + return PathGetPropertiesResponse(dgpr).ContentType() } // Date returns the value for header Date. func (dgpr DirectoryGetPropertiesResponse) Date() string { - return GetPathPropertiesResponse(dgpr).Date() + return PathGetPropertiesResponse(dgpr).Date() } // ETag returns the value for header ETag. func (dgpr DirectoryGetPropertiesResponse) ETag() string { - return GetPathPropertiesResponse(dgpr).ETag() + return PathGetPropertiesResponse(dgpr).ETag() } // LastModified returns the value for header Last-Modified. func (dgpr DirectoryGetPropertiesResponse) LastModified() string { - return GetPathPropertiesResponse(dgpr).LastModified() + return PathGetPropertiesResponse(dgpr).LastModified() } // XMsLeaseDuration returns the value for header x-ms-lease-duration. func (dgpr DirectoryGetPropertiesResponse) XMsLeaseDuration() string { - return GetPathPropertiesResponse(dgpr).XMsLeaseDuration() + return PathGetPropertiesResponse(dgpr).XMsLeaseDuration() } // XMsLeaseState returns the value for header x-ms-lease-state. func (dgpr DirectoryGetPropertiesResponse) XMsLeaseState() string { - return GetPathPropertiesResponse(dgpr).XMsLeaseState() + return PathGetPropertiesResponse(dgpr).XMsLeaseState() } // XMsLeaseStatus returns the value for header x-ms-lease-status. func (dgpr DirectoryGetPropertiesResponse) XMsLeaseStatus() string { - return GetPathPropertiesResponse(dgpr).XMsLeaseStatus() + return PathGetPropertiesResponse(dgpr).XMsLeaseStatus() } // XMsProperties returns the value for header x-ms-properties. func (dgpr DirectoryGetPropertiesResponse) XMsProperties() string { - return GetPathPropertiesResponse(dgpr).XMsProperties() + return PathGetPropertiesResponse(dgpr).XMsProperties() } // XMsRequestID returns the value for header x-ms-request-id. func (dgpr DirectoryGetPropertiesResponse) XMsRequestID() string { - return GetPathPropertiesResponse(dgpr).XMsRequestID() + return PathGetPropertiesResponse(dgpr).XMsRequestID() } // XMsResourceType returns the value for header x-ms-resource-type. func (dgpr DirectoryGetPropertiesResponse) XMsResourceType() string { - return GetPathPropertiesResponse(dgpr).XMsResourceType() + return PathGetPropertiesResponse(dgpr).XMsResourceType() } // XMsVersion returns the value for header x-ms-version. func (dgpr DirectoryGetPropertiesResponse) XMsVersion() string { - return GetPathPropertiesResponse(dgpr).XMsVersion() + return PathGetPropertiesResponse(dgpr).XMsVersion() } // DirectoryListResponse is the ListSchema response type. This type declaration is used to implement useful methods on // ListPath response -type DirectoryListResponse ListSchema +type DirectoryListResponse PathList // TODO: Used to by ListPathResponse. Have I changed it to the right thing? // Response returns the raw HTTP response object. func (dlr DirectoryListResponse) Response() *http.Response { - return ListSchema(dlr).Response() + return PathList(dlr).Response() } // StatusCode returns the HTTP status code of the response, e.g. 200. func (dlr DirectoryListResponse) StatusCode() int { - return ListSchema(dlr).StatusCode() + return PathList(dlr).StatusCode() } // Status returns the HTTP status message of the response, e.g. "200 OK". func (dlr DirectoryListResponse) Status() string { - return ListSchema(dlr).Status() + return PathList(dlr).Status() } // Date returns the value for header Date. func (dlr DirectoryListResponse) Date() string { - return ListSchema(dlr).Date() + return PathList(dlr).Date() } // ETag returns the value for header ETag. func (dlr DirectoryListResponse) ETag() string { - return ListSchema(dlr).ETag() + return PathList(dlr).ETag() } // LastModified returns the value for header Last-Modified. func (dlr DirectoryListResponse) LastModified() string { - return ListSchema(dlr).LastModified() + return PathList(dlr).LastModified() } // XMsContinuation returns the value for header x-ms-continuation. func (dlr DirectoryListResponse) XMsContinuation() string { - return ListSchema(dlr).XMsContinuation() + return PathList(dlr).XMsContinuation() } // XMsRequestID returns the value for header x-ms-request-id. func (dlr DirectoryListResponse) XMsRequestID() string { - return ListSchema(dlr).XMsRequestID() + return PathList(dlr).XMsRequestID() } // XMsVersion returns the value for header x-ms-version. func (dlr DirectoryListResponse) XMsVersion() string { - return ListSchema(dlr).XMsVersion() + return PathList(dlr).XMsVersion() } // Files returns the slice of all Files in ListDirectorySegment Response. // It does not include the sub-directory path -func (dlr *DirectoryListResponse) Files() []ListEntrySchema { - files := []ListEntrySchema{} - lSchema := ListSchema(*dlr) +func (dlr *DirectoryListResponse) Files() []Path { + files := []Path{} + lSchema := PathList(*dlr) for _, path := range lSchema.Paths { if path.IsDirectory != nil && *path.IsDirectory { continue @@ -277,7 +277,7 @@ func (dlr *DirectoryListResponse) Files() []ListEntrySchema { // It does not include the files inside the directory only returns the sub-directories func (dlr *DirectoryListResponse) Directories() []string { var dir []string - lSchema := (ListSchema)(*dlr) + lSchema := (PathList)(*dlr) for _, path := range lSchema.Paths { if path.IsDirectory == nil || (path.IsDirectory != nil && !*path.IsDirectory) { continue @@ -287,9 +287,9 @@ func (dlr *DirectoryListResponse) Directories() []string { return dir } -func (dlr *DirectoryListResponse) FilesAndDirectories() []ListEntrySchema { - var entities []ListEntrySchema - lSchema := (ListSchema)(*dlr) +func (dlr *DirectoryListResponse) FilesAndDirectories() []Path { + var entities []Path + lSchema := (PathList)(*dlr) for _, path := range lSchema.Paths { entities = append(entities, path) } @@ -298,7 +298,7 @@ func (dlr *DirectoryListResponse) FilesAndDirectories() []ListEntrySchema { // DownloadResponse wraps AutoRest generated downloadResponse and helps to provide info for retry. type DownloadResponse struct { - dr *ReadPathResponse + dr *ReadResponse // Fields need for retry. ctx context.Context @@ -347,7 +347,7 @@ func (dr DownloadResponse) ContentLanguage() string { } // ContentLength returns the value for header Content-Length. -func (dr DownloadResponse) ContentLength() string { +func (dr DownloadResponse) ContentLength() int64 { return dr.dr.ContentLength() } diff --git a/cmd/copyDownloadBlobFSEnumerator.go b/cmd/copyDownloadBlobFSEnumerator.go index 7879a00d8..ab86af77f 100644 --- a/cmd/copyDownloadBlobFSEnumerator.go +++ b/cmd/copyDownloadBlobFSEnumerator.go @@ -11,8 +11,6 @@ import ( "strings" - "strconv" - "github.com/Azure/azure-storage-azcopy/azbfs" "github.com/Azure/azure-storage-azcopy/common" ) @@ -55,10 +53,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { destination = cca.destination } - fileSize, err := strconv.ParseInt(props.ContentLength(), 10, 64) - if err != nil { - panic(err) - } + fileSize := props.ContentLength() // Queue the transfer e.addTransfer(common.CopyTransfer{ @@ -100,10 +95,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { fileURL := azbfs.NewFileURL(tempURLPartsExtension.URL(), p) if fileProperties, err := fileURL.GetProperties(ctx); err == nil && strings.EqualFold(fileProperties.XMsResourceType(), "file") { // file exists - fileSize, err := strconv.ParseInt(fileProperties.ContentLength(), 10, 64) - if err != nil { - panic(err) - } + fileSize := fileProperties.ContentLength() // assembling the file relative path fileRelativePath := fileOrDir @@ -133,10 +125,10 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { err := enumerateFilesInADLSGen2Directory( ctx, dirURL, - func(fileItem azbfs.ListEntrySchema) bool { // filter always return true in this case + func(fileItem azbfs.Path) bool { // filter always return true in this case return true }, - func(fileItem azbfs.ListEntrySchema) error { + func(fileItem azbfs.Path) error { relativePath := strings.Replace(*fileItem.Name, parentSourcePath, "", 1) if len(relativePath) > 0 && relativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR { relativePath = relativePath[1:] diff --git a/cmd/copyEnumeratorHelper.go b/cmd/copyEnumeratorHelper.go index d1f93c21e..a0b19c747 100644 --- a/cmd/copyEnumeratorHelper.go +++ b/cmd/copyEnumeratorHelper.go @@ -219,8 +219,8 @@ func enumerateDirectoriesAndFilesInShare(ctx context.Context, srcDirURL azfile.D ////////////////////////////////////////////////////////////////////////////////////////// // enumerateFilesInADLSGen2Directory enumerates files in ADLS Gen2 directory. func enumerateFilesInADLSGen2Directory(ctx context.Context, directoryURL azbfs.DirectoryURL, - filter func(fileItem azbfs.ListEntrySchema) bool, - callback func(fileItem azbfs.ListEntrySchema) error) error { + filter func(fileItem azbfs.Path) bool, + callback func(fileItem azbfs.Path) error) error { marker := "" for { listDirResp, err := directoryURL.ListDirectorySegment(ctx, &marker, true) From cd92186045d87d239df4b9883061fe18bf06f760 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 14:56:44 +1300 Subject: [PATCH 19/64] Copy the Swagger file to a copy that will be manually edited --- azbfs/azure_dfs_swagger_manually_edited.json | 1885 ++++++++++++++++++ 1 file changed, 1885 insertions(+) create mode 100644 azbfs/azure_dfs_swagger_manually_edited.json diff --git a/azbfs/azure_dfs_swagger_manually_edited.json b/azbfs/azure_dfs_swagger_manually_edited.json new file mode 100644 index 000000000..532cdb88b --- /dev/null +++ b/azbfs/azure_dfs_swagger_manually_edited.json @@ -0,0 +1,1885 @@ +{ + "swagger": "2.0", + "info": { + "description": "Azure Data Lake Storage provides storage for Hadoop and other big data workloads.", + "title": "Azure Data Lake Storage REST API", + "version": "2018-11-09", + "x-ms-code-generation-settings": { + "internalConstructors": true, + "name": "DataLakeStorageClient" + } + }, + "x-ms-parameterized-host": { + "hostTemplate": "{accountName}.{dnsSuffix}", + "parameters": [ + { + "$ref": "#/parameters/accountName" + }, + { + "$ref": "#/parameters/dnsSuffix" + } + ] + }, + "schemes": [ + "http", + "https" + ], + "produces": [ + "application/json" + ], + "tags": [ + { + "name": "Account Operations" + }, + { + "name": "Filesystem Operations" + }, + { + "name": "File and Directory Operations" + } + ], + "parameters": { + "Version": { + "description": "Specifies the version of the REST protocol used for processing the request. This is required when using shared key authorization.", + "in": "header", + "name": "x-ms-version", + "required": false, + "type": "string", + "x-ms-parameter-location": "client" + }, + "accountName": { + "description": "The Azure Storage account name.", + "in": "path", + "name": "accountName", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + }, + "dnsSuffix": { + "default": "dfs.core.windows.net", + "description": "The DNS suffix for the Azure Data Lake Storage endpoint.", + "in": "path", + "name": "dnsSuffix", + "required": true, + "type": "string", + "x-ms-skip-url-encoding": true, + "x-ms-parameter-location": "client" + } + }, + "definitions": { + "DataLakeStorageError": { + "properties": { + "error": { + "description": "The service error response object.", + "properties": { + "code": { + "description": "The service error code.", + "type": "string" + }, + "message": { + "description": "The service error message.", + "type": "string" + } + } + } + } + }, + "Path": { + "properties": { + "name": { + "type": "string" + }, + "isDirectory": { + "default": false, + "type": "boolean" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + }, + "contentLength": { + "type": "integer", + "format": "int64" + }, + "owner": { + "type": "string" + }, + "group": { + "type": "string" + }, + "permissions": { + "type": "string" + } + } + }, + "PathList": { + "properties": { + "paths": { + "type": "array", + "items": { + "$ref": "#/definitions/Path" + } + } + } + }, + "Filesystem": { + "properties": { + "name": { + "type": "string" + }, + "lastModified": { + "type": "string" + }, + "eTag": { + "type": "string" + } + } + }, + "FilesystemList": { + "properties": { + "filesystems": { + "type": "array", + "items": { + "$ref": "#/definitions/Filesystem" + } + } + } + } + }, + "responses": { + "ErrorResponse": { + "description": "An error occurred. The possible HTTP status, code, and message strings are listed below:\n* 400 Bad Request, ContentLengthMustBeZero, \"The Content-Length request header must be zero.\"\n* 400 Bad Request, InvalidAuthenticationInfo, \"Authentication information is not given in the correct format. Check the value of Authorization header.\"\n* 400 Bad Request, InvalidFlushPosition, \"The uploaded data is not contiguous or the position query parameter value is not equal to the length of the file after appending the uploaded data.\"\n* 400 Bad Request, InvalidHeaderValue, \"The value for one of the HTTP headers is not in the correct format.\"\n* 400 Bad Request, InvalidHttpVerb, \"The HTTP verb specified is invalid - it is not recognized by the server.\"\n* 400 Bad Request, InvalidInput, \"One of the request inputs is not valid.\"\n* 400 Bad Request, InvalidPropertyName, \"A property name cannot be empty.\"\n* 400 Bad Request, InvalidPropertyName, \"The property name contains invalid characters.\"\n* 400 Bad Request, InvalidQueryParameterValue, \"Value for one of the query parameters specified in the request URI is invalid.\"\n* 400 Bad Request, InvalidResourceName, \"The specified resource name contains invalid characters.\"\n* 400 Bad Request, InvalidSourceUri, \"The source URI is invalid.\"\n* 400 Bad Request, InvalidUri, \"The request URI is invalid.\"\n* 400 Bad Request, MissingRequiredHeader, \"An HTTP header that's mandatory for this request is not specified.\"\n* 400 Bad Request, MissingRequiredQueryParameter, \"A query parameter that's mandatory for this request is not specified.\"\n* 400 Bad Request, MultipleConditionHeadersNotSupported, \"Multiple condition headers are not supported.\"\n* 400 Bad Request, OutOfRangeInput, \"One of the request inputs is out of range.\"\n* 400 Bad Request, OutOfRangeQueryParameterValue, \"One of the query parameters specified in the request URI is outside the permissible range.\"\n* 400 Bad Request, UnsupportedHeader, \"One of the headers specified in the request is not supported.\"\n* 400 Bad Request, UnsupportedQueryParameter, \"One of the query parameters specified in the request URI is not supported.\"\n* 400 Bad Request, UnsupportedRestVersion, \"The specified Rest Version is Unsupported.\"\n* 403 Forbidden, AccountIsDisabled, \"The specified account is disabled.\"\n* 403 Forbidden, AuthorizationFailure, \"This request is not authorized to perform this operation.\"\n* 403 Forbidden, InsufficientAccountPermissions, \"The account being accessed does not have sufficient permissions to execute this operation.\"\n* 404 Not Found, FilesystemNotFound, \"The specified filesystem does not exist.\"\n* 404 Not Found, PathNotFound, \"The specified path does not exist.\"\n* 404 Not Found, RenameDestinationParentPathNotFound, \"The parent directory of the destination path does not exist.\"\n* 404 Not Found, ResourceNotFound, \"The specified resource does not exist.\"\n* 404 Not Found, SourcePathNotFound, \"The source path for a rename operation does not exist.\"\n* 405 Method Not Allowed, UnsupportedHttpVerb, \"The resource doesn't support the specified HTTP verb.\"\n* 409 Conflict, DestinationPathIsBeingDeleted, \"The specified destination path is marked to be deleted.\"\n* 409 Conflict, DirectoryNotEmpty, \"The recursive query parameter value must be true to delete a non-empty directory.\"\n* 409 Conflict, FilesystemAlreadyExists, \"The specified filesystem already exists.\"\n* 409 Conflict, FilesystemBeingDeleted, \"The specified filesystem is being deleted.\"\n* 409 Conflict, InvalidDestinationPath, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"* 409 Conflict, InvalidFlushOperation, \"The resource was created or modified by the Blob Service API and cannot be written to by the Data Lake Storage Service API.\"\n* 409 Conflict, InvalidRenameSourcePath, \"The source directory cannot be the same as the destination directory, nor can the destination be a subdirectory of the source directory.\"\n* 409 Conflict, InvalidSourceOrDestinationResourceType, \"The source and destination resource type must be identical.\"\n* 409 Conflict, LeaseAlreadyPresent, \"There is already a lease present.\"\n* 409 Conflict, LeaseIdMismatchWithLeaseOperation, \"The lease ID specified did not match the lease ID for the resource with the specified lease operation.\"\n* 409 Conflict, LeaseIsAlreadyBroken, \"The lease has already been broken and cannot be broken again.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeAcquired, \"The lease ID matched, but the lease is currently in breaking state and cannot be acquired until it is broken.\"\n* 409 Conflict, LeaseIsBreakingAndCannotBeChanged, \"The lease ID matched, but the lease is currently in breaking state and cannot be changed.\"\n* 409 Conflict, LeaseIsBrokenAndCannotBeRenewed, \"The lease ID matched, but the lease has been broken explicitly and cannot be renewed.\"\n* 409 Conflict, LeaseNameMismatch, \"The lease name specified did not match the existing lease name.\"\n* 409 Conflict, LeaseNotPresentWithLeaseOperation, \"The lease ID is not present with the specified lease operation.\"\n* 409 Conflict, PathAlreadyExists, \"The specified path already exists.\"\n* 409 Conflict, PathConflict, \"The specified path, or an element of the path, exists and its resource type is invalid for this operation.\"\n* 409 Conflict, SourcePathIsBeingDeleted, \"The specified source path is marked to be deleted.\"\n* 409 Conflict, ResourceTypeMismatch, \"The resource type specified in the request does not match the type of the resource.\"\n* 412 Precondition Failed, ConditionNotMet, \"The condition specified using HTTP conditional header(s) is not met.\"\n* 412 Precondition Failed, LeaseIdMismatch, \"The lease ID specified did not match the lease ID for the resource.\"\n* 412 Precondition Failed, LeaseIdMissing, \"There is currently a lease on the resource and no lease ID was specified in the request.\"\n* 412 Precondition Failed, LeaseNotPresent, \"There is currently no lease on the resource.\"\n* 412 Precondition Failed, LeaseLost, \"A lease ID was specified, but the lease for the resource has expired.\"\n* 412 Precondition Failed, SourceConditionNotMet, \"The source condition specified using HTTP conditional header(s) is not met.\"\n* 413 Request Entity Too Large, RequestBodyTooLarge, \"The request body is too large and exceeds the maximum permissible limit.\"\n* 416 Requested Range Not Satisfiable, InvalidRange, \"The range specified is invalid for the current size of the resource.\"\n* 500 Internal Server Error, InternalError, \"The server encountered an internal error. Please retry the request.\"\n* 500 Internal Server Error, OperationTimedOut, \"The operation could not be completed within the permitted time.\"\n* 503 Service Unavailable, ServerBusy, \"Egress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Ingress is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"Operations per second is over the account limit.\"\n* 503 Service Unavailable, ServerBusy, \"The server is currently unable to receive requests. Please retry your request.\"", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/DataLakeStorageError" + } + } + }, + "paths": { + "/": { + "get": { + "operationId": "Filesystem_List", + "summary": "List Filesystems", + "description": "List filesystems and their properties in given account.", + "x-ms-pageable": { + "itemName": "filesystems", + "nextLinkName": null + }, + "tags": [ + "Account Operations" + ], + "responses": { + "200": { + "description": "OK", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of filesystems to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "type": "string" + }, + "Content-Type": { + "description": "The content type of list filesystem response. The default content type is application/json.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/FilesystemList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "The value must be \"account\" for all account operations.", + "required": true, + "type": "string", + "enum": [ + "account" + ], + "x-ms-enum": { + "name": "AccountResourceType", + "modelAsString": false + } + }, + { + "name": "prefix", + "in": "query", + "description": "Filters results to filesystems within the specified prefix.", + "required": false, + "type": "string" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of filesystems returned with each invocation is limited. If the number of filesystems to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the filesystems.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + }, + "/{filesystem}": { + "put": { + "operationId": "Filesystem_Create", + "summary": "Create Filesystem", + "description": "Create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. This operation does not support conditional HTTP requests.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "201": { + "description": "Created", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Operations on files and directories do not affect the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Filesystem_SetProperties", + "summary": "Set Filesystem Properties", + "description": "Set properties for the filesystem. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the filesystem, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. If the filesystem exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Path_List", + "summary": "List Paths", + "description": "List filesystem paths and their properties.", + "x-ms-pageable": { + "itemName": "paths", + "nextLinkName": null + }, + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "If the number of paths to be listed exceeds the maxResults limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "type": "string" + } + }, + "schema": { + "$ref": "#/definitions/PathList" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "directory", + "in": "query", + "description": "Filters results to paths within the specified directory. An error occurs if the directory does not exist.", + "required": false, + "type": "string" + }, + { + "name": "recursive", + "in": "query", + "description": "If \"true\", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If \"directory\" is specified, the list will only include paths that share the same root.", + "required": true, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "The number of paths returned with each invocation is limited. If the number of paths to be returned exceeds this limit, a continuation token is returned in the response header x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the list operation to continue listing the paths.", + "required": false, + "type": "string" + }, + { + "name": "maxResults", + "in": "query", + "description": "An optional value that specifies the maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + } + ] + }, + "head": { + "operationId": "Filesystem_GetProperties", + "summary": "Get Filesystem Properties.", + "description": "All system and user-defined filesystem properties are specified in the response headers.", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the filesystem. Changes to filesystem properties affect the entity tag, but operations on files and directories do not.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the filesystem was last modified. Changes to filesystem properties update the last modified time, but operations on files and directories do not.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the filesystem. A comma-separated list of name and value pairs in the format \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-namespace-enabled": { + "description": "A bool string indicates whether the namespace feature is enabled. If \"true\", the namespace is enabled for the filesystem. ", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + } + }, + "delete": { + "operationId": "Filesystem_Delete", + "summary": "Delete Filesystem", + "description": "Marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier cannot be created for at least 30 seconds. While the filesystem is being deleted, attempts to create a filesystem with the same identifier will fail with status code 409 (Conflict), with the service returning additional error information indicating that the filesystem is being deleted. All other operations, including operations on any files or directories within the filesystem, will fail with status code 404 (Not Found) while the filesystem is being deleted. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "Filesystem Operations" + ], + "responses": { + "202": { + "description": "Accepted", + "headers": { + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have between 3 and 63 characters.", + "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "resource", + "in": "query", + "description": "The value must be \"filesystem\" for all filesystem operations.", + "required": true, + "type": "string", + "enum": [ + "filesystem" + ], + "x-ms-enum": { + "name": "FilesystemResourceType", + "modelAsString": false + } + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + }, + "/{filesystem}/{path}": { + "put": { + "operationId": "Path_Create", + "summary": "Create File | Create Directory | Rename File | Rename Directory", + "description": "Create or rename a file or directory. By default, the destination is overwritten and if the destination already exists and has a lease the lease is broken. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). To fail if the destination already exists, use a conditional request with If-None-Match: \"*\".", + "consumes": [ + "application/octet-stream" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "201": { + "description": "The file or directory was created.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "resource", + "in": "query", + "description": "Required only for Create File and Create Directory. The value must be \"file\" or \"directory\".", + "required": false, + "type": "string", + "enum": [ + "directory", + "file" + ], + "x-ms-enum": { + "name": "PathResourceType", + "modelAsString": false + } + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When renaming a directory, the number of paths that are renamed with each invocation is limited. If the number of paths to be renamed exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the rename operation to continue renaming the directory.", + "required": false, + "type": "string" + }, + { + "name": "mode", + "in": "query", + "description": "Optional. Valid only when namespace is enabled. This parameter determines the behavior of the rename operation. The value must be \"legacy\" or \"posix\", and the default value will be \"posix\". ", + "required": false, + "type": "string", + "enum": [ + "legacy", + "posix" + ], + "x-ms-enum": { + "name": "PathRenameMode", + "modelAsString": false + } + }, + { + "name": "Cache-Control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "Content-Encoding", + "in": "header", + "description": "Optional. Specifies which content encodings have been applied to the file. This value is returned to the client when the \"Read File\" operation is performed.", + "required": false, + "type": "string" + }, + { + "name": "Content-Language", + "in": "header", + "description": "Optional. Specifies the natural language used by the intended audience for the file.", + "required": false, + "type": "string" + }, + { + "name": "Content-Disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-rename-source", + "in": "header", + "description": "An optional file or directory to be renamed. The value must have the following format: \"/{filesystem}/{path}\". If \"x-ms-properties\" is specified, the properties will overwrite the existing properties; otherwise, the existing properties will be preserved. This value must be a URL percent-encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. A lease ID for the path specified in the URI. The path to be overwritten must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-lease-id", + "in": "header", + "description": "Optional for rename operations. A lease ID for the source path. The source path must have an active lease and the lease ID must match.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-umask", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. When creating a file or directory and the parent folder does not have a default ACL, the umask restricts the permissions of the file or directory to be created. The resulting permission is given by p & ^u, where p is the permission and u is the umask. For example, if p is 0777 and u is 0057, then the resulting permission is 0720. The default permission is 0777 for a directory and 0666 for a file. The default umask is 0027. The umask must be specified in 4-digit octal notation (e.g. 0766).", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-match", + "description": "Optional. An ETag value. Specify this header to perform the rename operation only if the source's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-none-match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the rename operation only if the source's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-modified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-source-if-unmodified-since", + "description": "Optional. A date and time value. Specify this header to perform the rename operation only if the source has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "patch": { + "operationId": "Path_Update", + "summary": "Append Data | Flush Data | Set Properties | Set Access Control", + "description": "Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "consumes": [ + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The data was flushed (written) to the file or the properties were set successfully.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "x-ms-properties": { + "description": "User-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "202": { + "description": "The uploaded data was accepted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "The action must be \"append\" to upload data to be appended to a file, \"flush\" to flush previously uploaded data to a file, \"setProperties\" to set the properties of a file or directory, or \"setAccessControl\" to set the owner, group, permissions, or access control list for a file or directory. Note that Hierarchical Namespace must be enabled for the account in order to use access control. Also note that the Access Control List (ACL) includes permissions for the owner, owning group, and others, so the x-ms-permissions and x-ms-acl request headers are mutually exclusive.", + "required": true, + "type": "string", + "enum": [ + "append", + "flush", + "setProperties", + "setAccessControl" + ], + "x-ms-enum": { + "name": "PathUpdateAction", + "modelAsString": false + } + }, + { + "name": "position", + "in": "query", + "description": "This parameter allows the caller to upload data in parallel and control the order in which it is appended to the file. It is required when uploading data to be appended to the file and when flushing previously uploaded data to the file. The value must be the position where the data is to be appended. Uploaded data is not immediately flushed, or written, to the file. To flush, the previously uploaded data must be contiguous, the position parameter must be specified and equal to the length of the file after all data has been written, and there must not be a request entity body included with the request.", + "format": "int64", + "required": false, + "type": "integer" + }, + { + "name": "retainUncommittedData", + "in": "query", + "description": "Valid only for flush operations. If \"true\", uncommitted data is retained after the flush operation completes; otherwise, the uncommitted data is deleted after the flush operation. The default is false. Data at offsets less than the specified position are written to the file when flush succeeds, but this optional parameter allows data after the flush position to be retained for a future flush operation.", + "required": false, + "type": "boolean" + }, + { + "name": "close", + "in": "query", + "description": "Azure Storage Events allow applications to receive notifications when files change. When Azure Storage Events are enabled, a file changed event is raised. This event has a property indicating whether this is the final change to distinguish the difference between an intermediate flush to a file stream and the final close of a file stream. The close query parameter is valid only when the action is \"flush\" and change notifications are enabled. If the value of close is \"true\" and the flush operation completes successfully, the service raises a file change notification with a property indicating that this is the final update (the file stream has been closed). If \"false\" a change notification is raised indicating the file has changed. The default is false. This query parameter is set to true by the Hadoop ABFS driver to indicate that the file stream has been closed.\"", + "required": false, + "type": "boolean" + }, + { + "name": "Content-Length", + "in": "header", + "description": "Required for \"Append Data\" and \"Flush Data\". Must be 0 for \"Flush Data\". Must be the length of the request content in bytes for \"Append Data\".", + "minimum": 0, + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-cache-control", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Cache-Control\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-type", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Type\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-disposition", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Disposition\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-encoding", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Encoding\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-language", + "in": "header", + "description": "Optional and only valid for flush and set properties operations. The service stores this value and includes it in the \"Content-Language\" response header for \"Read File\" operations.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-content-md5", + "in": "header", + "description": "Optional and only valid for \"Flush & Set Properties\" operations. The service stores this value and includes it in the \"Content-Md5\" response header for \"Read & Get Properties\" operations. If this property is not specified on the request, then the property will be cleared for the file. Subsequent calls to \"Read & Get Properties\" will not return this property unless it is explicitly set on that file again.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-properties", + "description": "Optional. User-defined properties to be stored with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set. Valid only for the setProperties operation. If the file or directory exists, any properties not included in the list will be removed. All properties are removed if the header is omitted. To merge new and existing properties, first get all existing properties and the current E-Tag, then make a conditional request with the E-Tag and include values for all properties.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-owner", + "description": "Optional and valid only for the setAccessControl operation. Sets the owner of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-group", + "description": "Optional and valid only for the setAccessControl operation. Sets the owning group of the file or directory.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-permissions", + "description": "Optional and only valid if Hierarchical Namespace is enabled for the account. Sets POSIX access permissions for the file owner, the file owning group, and others. Each class may be granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) and 4-digit octal notation (e.g. 0766) are supported. Invalid in conjunction with x-ms-acl.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "x-ms-acl", + "description": "Optional and valid only for the setAccessControl operation. Sets POSIX access control rights on files and directories. The value is a comma-separated list of access control entries that fully replaces the existing access control list (ACL). Each access control entry (ACE) consists of a scope, a type, a user or group identifier, and permissions in the format \"[scope:][type]:[id]:[permissions]\". The scope must be \"default\" to indicate the ACE belongs to the default ACL for a directory; otherwise scope is implicit and the ACE belongs to the access ACL. There are four ACE types: \"user\" grants rights to the owner or a named user, \"group\" grants rights to the owning group or a named group, \"mask\" restricts rights granted to named users and the members of groups, and \"other\" grants rights to all users not found in any of the other entries. The user or group identifier is omitted for entries of type \"mask\" and \"other\". The user or group identifier is also omitted for the owner and owning group. The permission field is a 3-character sequence where the first character is 'r' to grant read access, the second character is 'w' to grant write access, and the third character is 'x' to grant execute permission. If access is not granted, the '-' character is used to denote that the permission is denied. For example, the following ACL grants read, write, and execute rights to the file owner and john.doe@contoso, the read right to the owning group, and nothing to everyone else: \"user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask=rwx\". Invalid in conjunction with x-ms-permissions.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional for Flush Data and Set Properties, but invalid for Append Data. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "requestBody", + "description": "Valid only for append operations. The data to be uploaded and appended to the file.", + "in": "body", + "required": false, + "schema": { + "type": "object", + "format": "file" + } + } + ] + }, + "post": { + "operationId": "Path_Lease", + "summary": "Lease Path", + "description": "Create and manage a lease to restrict write and delete access to the path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The \"renew\", \"change\" or \"release\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file was last modified. Write operations on the file update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"renew\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "201": { + "description": "A new lease has been created. The \"acquire\" action was successful.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-id": { + "description": "A successful \"acquire\" action returns the lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + } + } + }, + "202": { + "description": "The \"break\" lease action was successful.", + "headers": { + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-lease-time": { + "description": "The time remaining in the lease period in seconds.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "x-ms-lease-action", + "in": "header", + "description": "There are five lease actions: \"acquire\", \"break\", \"change\", \"renew\", and \"release\". Use \"acquire\" and specify the \"x-ms-proposed-lease-id\" and \"x-ms-lease-duration\" to acquire a new lease. Use \"break\" to break an existing lease. When a lease is broken, the lease break period is allowed to elapse, during which time no lease operation except break and release can be performed on the file. When a lease is successfully broken, the response indicates the interval in seconds until a new lease can be acquired. Use \"change\" and specify the current lease ID in \"x-ms-lease-id\" and the new lease ID in \"x-ms-proposed-lease-id\" to change the lease ID of an active lease. Use \"renew\" and specify the \"x-ms-lease-id\" to renew an existing lease. Use \"release\" and specify the \"x-ms-lease-id\" to release a lease.", + "required": true, + "type": "string", + "enum": [ + "acquire", + "break", + "change", + "renew", + "release" + ], + "x-ms-enum": { + "name": "PathLeaseAction", + "modelAsString": false + } + }, + { + "name": "x-ms-lease-duration", + "in": "header", + "description": "The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-break-period", + "in": "header", + "description": "The lease break period duration is optional to break a lease, and specifies the break period of the lease in seconds. The lease break duration must be between 0 and 60 seconds.", + "format": "int32", + "required": false, + "type": "integer" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"renew\", \"change\" or \"release\". For the renew and release actions, this must match the current lease ID.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "x-ms-proposed-lease-id", + "in": "header", + "description": "Required when \"x-ms-lease-action\" is \"acquire\" or \"change\". A lease will be acquired with this lease ID if the operation is successful.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "get": { + "operationId": "Path_Read", + "summary": "Read File", + "description": "Read the contents of a file. For read operations, range requests are supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "produces": [ + "application/json", + "application/octet-stream", + "text/plain" + ], + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Ok", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file. If the file has an MD5 hash and this read operation is to read the complete file, this response header is returned so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "206": { + "description": "Partial content", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + }, + "schema": { + "type": "file" + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "in": "header", + "description": "The HTTP Range request header specifies one or more byte ranges of the resource to be retrieved.", + "required": false, + "type": "string", + "name": "Range" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "head": { + "operationId": "Path_GetProperties", + "summary": "Get Properties | Get Status | Get Access Control List", + "description": "Get Properties returns all system and user defined properties for a path. Get Status returns all system defined properties for a path. Get Access Control List returns the access control list for a path. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "Returns all properties for the file or directory.", + "headers": { + "Accept-Ranges": { + "description": "Indicates that the service supports requests for partial file content.", + "type": "string" + }, + "Cache-Control": { + "description": "If the Cache-Control request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Disposition": { + "description": "If the Content-Disposition request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Encoding": { + "description": "If the Content-Encoding request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Language": { + "description": "If the Content-Language request header has previously been set for the resource, that value is returned in this header.", + "type": "string" + }, + "Content-Length": { + "description": "The size of the resource in bytes.", + "type": "integer", + "format": "int64" + }, + "Content-Range": { + "description": "Indicates the range of bytes returned in the event that the client requested a subset of the file by setting the Range request header.", + "type": "string" + }, + "Content-Type": { + "description": "The content type specified for the resource. If no content type was specified, the default content type is application/octet-stream.", + "type": "string" + }, + "Content-MD5": { + "description": "The MD5 hash of complete file stored in storage. This header is returned only for \"GetProperties\" operation. If the Content-MD5 header has been set for the file, this response header is returned for GetProperties call so that the client can check for message content integrity.", + "type": "string" + }, + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "ETag": { + "description": "An HTTP entity tag associated with the file or directory.", + "type": "string" + }, + "Last-Modified": { + "description": "The data and time the file or directory was last modified. Write operations on the file or directory update the last modified time.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-resource-type": { + "description": "The type of the resource. The value may be \"file\" or \"directory\". If not set, the value is \"file\".", + "type": "string" + }, + "x-ms-properties": { + "description": "The user-defined properties associated with the file or directory, in the format of a comma-separated list of name and value pairs \"n1=v1, n2=v2, ...\", where each value is a base64 encoded string. Note that the string may only contain ASCII characters in the ISO-8859-1 character set.", + "type": "string" + }, + "x-ms-owner": { + "description": "The owner of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-group": { + "description": "The owning group of the file or directory. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-permissions": { + "description": "The POSIX access permissions for the file owner, the file owning group, and others. Included in the response if Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-acl": { + "description": "The POSIX access control list for the file or directory. Included in the response only if the action is \"getAccessControl\" and Hierarchical Namespace is enabled for the account.", + "type": "string" + }, + "x-ms-lease-duration": { + "description": "When a resource is leased, specifies whether the lease is of infinite or fixed duration.", + "type": "string" + }, + "x-ms-lease-state": { + "description": "Lease state of the resource. ", + "type": "string" + }, + "x-ms-lease-status": { + "description": "The lease status of the resource.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "action", + "in": "query", + "description": "Optional. If the value is \"getStatus\" only the system defined properties for the path are returned. If the value is \"getAccessControl\" the access control list is returned in the response headers (Hierarchical Namespace must be enabled for the account), otherwise the properties are returned.", + "required": false, + "type": "string", + "enum": [ + "getAccessControl", + "getStatus" + ], + "x-ms-enum": { + "name": "PathGetPropertiesAction", + "modelAsString": false + } + }, + { + "name": "upn", + "in": "query", + "description": "Optional. Valid only when Hierarchical Namespace is enabled for the account. If \"true\", the user identity values returned in the x-ms-owner, x-ms-group, and x-ms-acl response headers will be transformed from Azure Active Directory Object IDs to User Principal Names. If \"false\", the values will be returned as Azure Active Directory Object IDs. The default value is false. Note that group and application Object IDs are not translated because they do not have unique friendly names.", + "required": false, + "type": "boolean" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "Optional. If this header is specified, the operation will be performed only if both of the following conditions are met: i) the path's lease is currently active and ii) the lease ID specified in the request matches that of the path.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "delete": { + "operationId": "Path_Delete", + "summary": "Delete File | Delete Directory", + "description": "Delete the file or directory. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations).", + "tags": [ + "File and Directory Operations" + ], + "responses": { + "200": { + "description": "The file was deleted.", + "headers": { + "Date": { + "description": "A UTC date/time value generated by the service that indicates the time at which the response was initiated.", + "type": "string" + }, + "x-ms-request-id": { + "description": "A server-generated UUID recorded in the analytics logs for troubleshooting and correlation.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "type": "string" + }, + "x-ms-version": { + "description": "The version of the REST protocol used to process the request.", + "type": "string" + }, + "x-ms-continuation": { + "description": "When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "type": "string" + } + } + }, + "default": { + "$ref": "#/responses/ErrorResponse" + } + }, + "parameters": [ + { + "name": "recursive", + "in": "query", + "description": "Required and valid only when the resource is a directory. If \"true\", all paths beneath the directory will be deleted. If \"false\" and the directory is non-empty, an error occurs.", + "required": false, + "type": "boolean" + }, + { + "name": "continuation", + "in": "query", + "description": "Optional. When deleting a directory, the number of paths that are deleted with each invocation is limited. If the number of paths to be deleted exceeds this limit, a continuation token is returned in this response header. When a continuation token is returned in the response, it must be specified in a subsequent invocation of the delete operation to continue deleting the directory.", + "required": false, + "type": "string" + }, + { + "name": "x-ms-lease-id", + "in": "header", + "description": "The lease ID must be specified if there is an active lease.", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string" + }, + { + "name": "If-Match", + "description": "Optional. An ETag value. Specify this header to perform the operation only if the resource's ETag matches the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-None-Match", + "description": "Optional. An ETag value or the special wildcard (\"*\") value. Specify this header to perform the operation only if the resource's ETag does not match the value specified. The ETag must be specified in quotes.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Modified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + }, + { + "name": "If-Unmodified-Since", + "description": "Optional. A date and time value. Specify this header to perform the operation only if the resource has not been modified since the specified date and time.", + "in": "header", + "required": false, + "type": "string" + } + ] + }, + "parameters": [ + { + "name": "filesystem", + "in": "path", + "description": "The filesystem identifier.", + "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "minLength": 3, + "maxLength": 63, + "required": true, + "type": "string" + }, + { + "name": "path", + "in": "path", + "description": "The file or directory path.", + "required": true, + "type": "string" + }, + { + "name": "x-ms-client-request-id", + "description": "A UUID recorded in the analytics logs for troubleshooting and correlation.", + "in": "header", + "pattern": "^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$", + "required": false, + "type": "string", + "x-ms-client-request-id": true + }, + { + "name": "timeout", + "in": "query", + "description": "An optional operation timeout value in seconds. The period begins when the request is received by the service. If the timeout value elapses before the operation completes, the operation fails.", + "format": "int32", + "minimum": 1, + "required": false, + "type": "integer" + }, + { + "name": "x-ms-date", + "in": "header", + "description": "Specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key authorization.", + "required": false, + "type": "string" + }, + { + "$ref": "#/parameters/Version" + } + ] + } + } +} From 7bb6411706acb1b6cf9ba08d7ba4793d2a56be5b Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 14:58:09 +1300 Subject: [PATCH 20/64] Fix compilation error in generated code with regard to xMsVersion By putting version back to global and mandatory, but not with a "client" parameter location --- azbfs/azure_dfs_swagger_manually_edited.json | 7 +++-- azbfs/zz_generated_filesystem.go | 20 ++++---------- azbfs/zz_generated_path.go | 28 +++++--------------- 3 files changed, 15 insertions(+), 40 deletions(-) diff --git a/azbfs/azure_dfs_swagger_manually_edited.json b/azbfs/azure_dfs_swagger_manually_edited.json index 532cdb88b..08be2f741 100644 --- a/azbfs/azure_dfs_swagger_manually_edited.json +++ b/azbfs/azure_dfs_swagger_manually_edited.json @@ -43,10 +43,9 @@ "description": "Specifies the version of the REST protocol used for processing the request. This is required when using shared key authorization.", "in": "header", "name": "x-ms-version", - "required": false, - "type": "string", - "x-ms-parameter-location": "client" - }, + "required": true, + "type": "string" + }, "accountName": { "description": "The Azure Storage account name.", "in": "path", diff --git a/azbfs/zz_generated_filesystem.go b/azbfs/zz_generated_filesystem.go index a52000077..25d5b532d 100644 --- a/azbfs/zz_generated_filesystem.go +++ b/azbfs/zz_generated_filesystem.go @@ -89,9 +89,7 @@ func (client filesystemClient) createPreparer(filesystem string, xMsProperties * if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -176,9 +174,7 @@ func (client filesystemClient) deletePreparer(filesystem string, ifModifiedSince if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -247,9 +243,7 @@ func (client filesystemClient) getPropertiesPreparer(filesystem string, xMsClien if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -330,9 +324,7 @@ func (client filesystemClient) listPreparer(prefix *string, continuation *string if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -434,9 +426,7 @@ func (client filesystemClient) setPropertiesPreparer(filesystem string, xMsPrope if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } diff --git a/azbfs/zz_generated_path.go b/azbfs/zz_generated_path.go index 51360fd17..ef56684f8 100644 --- a/azbfs/zz_generated_path.go +++ b/azbfs/zz_generated_path.go @@ -219,9 +219,7 @@ func (client pathClient) createPreparer(filesystem string, pathParameter string, if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -326,9 +324,7 @@ func (client pathClient) deletePreparer(filesystem string, pathParameter string, if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -439,9 +435,7 @@ func (client pathClient) getPropertiesPreparer(filesystem string, pathParameter if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -562,9 +556,7 @@ func (client pathClient) leasePreparer(xMsLeaseAction PathLeaseActionType, files if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -662,9 +654,7 @@ func (client pathClient) listPreparer(recursive bool, filesystem string, directo if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -778,9 +768,7 @@ func (client pathClient) readPreparer(filesystem string, pathParameter string, r if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -990,9 +978,7 @@ func (client pathClient) updatePreparer(action PathUpdateActionType, filesystem if xMsDate != nil { req.Header.Set("x-ms-date", *xMsDate) } - if xMsVersion != nil { - req.Header.Set("x-ms-version", *client.XMsVersion) - } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } From cba1562eb239398a3566da9837ff7c9a7ffcd63c Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 15:01:09 +1300 Subject: [PATCH 21/64] Remove negative lookahead assertion from filesystem name validation Because negative lookaheads are not supported by the Go Regex engine --- azbfs/azure_dfs_swagger_manually_edited.json | 4 ++-- azbfs/zz_generated_filesystem.go | 8 ++++---- azbfs/zz_generated_path.go | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/azbfs/azure_dfs_swagger_manually_edited.json b/azbfs/azure_dfs_swagger_manually_edited.json index 08be2f741..dfbfccc77 100644 --- a/azbfs/azure_dfs_swagger_manually_edited.json +++ b/azbfs/azure_dfs_swagger_manually_edited.json @@ -586,7 +586,7 @@ "name": "filesystem", "in": "path", "description": "The filesystem identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have between 3 and 63 characters.", - "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "pattern": "^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$", "minLength": 3, "maxLength": 63, "required": true, @@ -1837,7 +1837,7 @@ "name": "filesystem", "in": "path", "description": "The filesystem identifier.", - "pattern": "^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$", + "pattern": "^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$", "minLength": 3, "maxLength": 63, "required": true, diff --git a/azbfs/zz_generated_filesystem.go b/azbfs/zz_generated_filesystem.go index 25d5b532d..7d4109a75 100644 --- a/azbfs/zz_generated_filesystem.go +++ b/azbfs/zz_generated_filesystem.go @@ -46,7 +46,7 @@ func (client filesystemClient) Create(ctx context.Context, filesystem string, xM { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -128,7 +128,7 @@ func (client filesystemClient) Delete(ctx context.Context, filesystem string, if { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -203,7 +203,7 @@ func (client filesystemClient) GetProperties(ctx context.Context, filesystem str { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -377,7 +377,7 @@ func (client filesystemClient) SetProperties(ctx context.Context, filesystem str { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, diff --git a/azbfs/zz_generated_path.go b/azbfs/zz_generated_path.go index ef56684f8..6426c5a60 100644 --- a/azbfs/zz_generated_path.go +++ b/azbfs/zz_generated_path.go @@ -102,7 +102,7 @@ func (client pathClient) Create(ctx context.Context, filesystem string, pathPara { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -264,7 +264,7 @@ func (client pathClient) Delete(ctx context.Context, filesystem string, pathPara { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -375,7 +375,7 @@ func (client pathClient) GetProperties(ctx context.Context, filesystem string, p { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -492,7 +492,7 @@ func (client pathClient) Lease(ctx context.Context, xMsLeaseAction PathLeaseActi { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -601,7 +601,7 @@ func (client pathClient) List(ctx context.Context, recursive bool, filesystem st { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -711,7 +711,7 @@ func (client pathClient) Read(ctx context.Context, filesystem string, pathParame { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, @@ -878,7 +878,7 @@ func (client pathClient) Update(ctx context.Context, action PathUpdateActionType { targetValue: filesystem, constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9](?!.*--)[-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, { targetValue: xMsClientRequestID, constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, From de134a6395c93171b4d3d0df886b0b213d6c48df Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 15:05:00 +1300 Subject: [PATCH 22/64] Re-introduce x-http-method-override for PATCH Required when using Go SDK --- azbfs/azure_dfs_swagger_manually_edited.json | 7 ++++++ azbfs/url_file.go | 20 +++++++---------- azbfs/zz_generated_path.go | 23 +++++++++++++++----- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/azbfs/azure_dfs_swagger_manually_edited.json b/azbfs/azure_dfs_swagger_manually_edited.json index dfbfccc77..90b652e4b 100644 --- a/azbfs/azure_dfs_swagger_manually_edited.json +++ b/azbfs/azure_dfs_swagger_manually_edited.json @@ -1150,6 +1150,13 @@ "type": "string" }, { + "name": "x-http-method-override", + "description": "Optional. Override the http verb on the service side. Some older http clients do not support PATCH", + "in": "header", + "required": false, + "type": "string" + }, + { "name": "requestBody", "description": "Valid only for append operations. The data to be uploaded and appended to the file.", "in": "body", diff --git a/azbfs/url_file.go b/azbfs/url_file.go index 916d08666..74818c42b 100644 --- a/azbfs/url_file.go +++ b/azbfs/url_file.go @@ -129,17 +129,15 @@ func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeke panic("body must contain readable data whose size is > 0") } - // TODO: does this AppendData function even work now? We use to override the Http verb, but the new API doesn't - // let us do that - // Old_TODO_text: the go http client has a problem with PATCH and content-length header + // TODO: the go http client has a problem with PATCH and content-length header // we should investigate and report the issue - // overrideHttpVerb := "PATCH" + overrideHttpVerb := "PATCH" // TransactionalContentMD5 isn't supported currently. return f.fileClient.Update(ctx, PathUpdateActionAppend, f.fileSystemName, f.path, &offset, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, body, nil, nil, nil) + nil, nil, nil, nil, nil, nil, &overrideHttpVerb, body, nil, nil, nil) } // flushes writes previously uploaded data to a file @@ -152,19 +150,17 @@ func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*PathUpdateResp // azcopy does not need this retainUncommittedData := false - // TODO: does this FlushData function even work now? We use to override the Http verb, but the new API doesn't - // let us do that - // Old_TODO_text: the go http client has a problem with PATCH and content-length header - // we should investigate and report the issue - // overrideHttpVerb := "PATCH" + // TODO: the go http client has a problem with PATCH and content-length header + // we should investigate and report the issue + overrideHttpVerb := "PATCH" // TODO: feb 2019 API update: review the use of closeParameter here. Should it be true? - // Doc implies only make it true if + // Doc implies only make it true if this is the end of the file // TransactionalContentMD5 isn't supported currently. return f.fileClient.Update(ctx, PathUpdateActionFlush, f.fileSystemName, f.path, &fileSize, &retainUncommittedData, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil) + nil, &overrideHttpVerb, nil, nil, nil, nil) } diff --git a/azbfs/zz_generated_path.go b/azbfs/zz_generated_path.go index 6426c5a60..ec743a731 100644 --- a/azbfs/zz_generated_path.go +++ b/azbfs/zz_generated_path.go @@ -859,13 +859,14 @@ func (client pathClient) readResponder(resp pipeline.Response) (pipeline.Respons // Specify this header to perform the operation only if the resource has been modified since the specified date and // time. ifUnmodifiedSince is optional for Flush Data and Set Properties, but invalid for Append Data. A date and time // value. Specify this header to perform the operation only if the resource has not been modified since the specified -// date and time. requestBody is valid only for append operations. The data to be uploaded and appended to the file. -// requestBody will be closed upon successful return. Callers should ensure closure when receiving an +// date and time. xHTTPMethodOverride is optional. Override the http verb on the service side. Some older http clients +// do not support PATCH requestBody is valid only for append operations. The data to be uploaded and appended to the +// file. requestBody will be closed upon successful return. Callers should ensure closure when receiving an // error.xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and correlation. timeout is an // optional operation timeout value in seconds. The period begins when the request is received by the service. If the // timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated // Universal Time (UTC) for the request. This is required when using shared key authorization. -func (client pathClient) Update(ctx context.Context, action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathUpdateResponse, error) { +func (client pathClient) Update(ctx context.Context, action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathUpdateResponse, error) { if err := validate([]validation{ { targetValue: contentLength, constraints: []constraint{ {target: "contentLength", name: null, rule: false , @@ -889,7 +890,7 @@ func (client pathClient) Update(ctx context.Context, action PathUpdateActionType }}}}}); err != nil { return nil, err } - req, err := client.updatePreparer(action, filesystem, pathParameter, position, retainUncommittedData, closeParameter, contentLength, xMsLeaseID, xMsCacheControl, xMsContentType, xMsContentDisposition, xMsContentEncoding, xMsContentLanguage, xMsContentMd5, xMsProperties, xMsOwner, xMsGroup, xMsPermissions, xMsACL, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, body, xMsClientRequestID, timeout, xMsDate) + req, err := client.updatePreparer(action, filesystem, pathParameter, position, retainUncommittedData, closeParameter, contentLength, xMsLeaseID, xMsCacheControl, xMsContentType, xMsContentDisposition, xMsContentEncoding, xMsContentLanguage, xMsContentMd5, xMsProperties, xMsOwner, xMsGroup, xMsPermissions, xMsACL, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xHTTPMethodOverride, body, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } @@ -901,8 +902,15 @@ func (client pathClient) Update(ctx context.Context, action PathUpdateActionType } // updatePreparer prepares the Update request. -func (client pathClient) updatePreparer(action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("PATCH", client.url, body) +func (client pathClient) updatePreparer(action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + // begin manual edit to generated code + method := "PATCH" + if xHTTPMethodOverride != nil { + method = "PUT" + } + req, err := pipeline.NewRequest(method, client.url, body) + // end manual edit to generated code + if err != nil { return req, pipeline.NewError(err, "failed to create request") } @@ -972,6 +980,9 @@ func (client pathClient) updatePreparer(action PathUpdateActionType, filesystem if ifUnmodifiedSince != nil { req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) } + if xHTTPMethodOverride != nil { + req.Header.Set("x-http-method-override", *xHTTPMethodOverride) + } if xMsClientRequestID != nil { req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) } From 22bd703f7ab2797b44ec2039d405661e9d256a79 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 17:21:36 +1300 Subject: [PATCH 23/64] Remove duplicates from autorest-generated imports --- azbfs/zz_generated_filesystem.go | 445 ++++++------- azbfs/zz_generated_path.go | 1062 +++++++++++++++--------------- 2 files changed, 732 insertions(+), 775 deletions(-) diff --git a/azbfs/zz_generated_filesystem.go b/azbfs/zz_generated_filesystem.go index 7d4109a75..e331811e3 100644 --- a/azbfs/zz_generated_filesystem.go +++ b/azbfs/zz_generated_filesystem.go @@ -4,29 +4,26 @@ package azbfs // Changes may cause incorrect behavior and will be lost if the code is regenerated. import ( - "net/url" - "github.com/Azure/azure-pipeline-go/pipeline" - "net/url" - "net/http" - "net/url" - "context" - "net/url" - "strconv" - "net/url" - "encoding/json" - "net/url" - "io/ioutil" - "net/url" - "io" + // begin manual edit to generated code + "context" + "encoding/json" + "github.com/Azure/azure-pipeline-go/pipeline" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + // end manual edit ) // filesystemClient is the azure Data Lake Storage provides storage for Hadoop and other big data workloads. type filesystemClient struct { - managementClient + managementClient } + // newFilesystemClient creates an instance of the filesystemClient client. func newFilesystemClient(url url.URL, p pipeline.Pipeline) filesystemClient { - return filesystemClient{newManagementClient(url, p)} + return filesystemClient{newManagementClient(url, p)} } // Create create a filesystem rooted at the specified location. If the filesystem already exists, the operation fails. @@ -42,29 +39,27 @@ func newFilesystemClient(url url.URL, p pipeline.Pipeline) filesystemClient { // timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated // Universal Time (UTC) for the request. This is required when using shared key authorization. func (client filesystemClient) Create(ctx context.Context, filesystem string, xMsProperties *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemCreateResponse, error) { - if err := validate([]validation{ - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.createPreparer(filesystem, xMsProperties, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*FilesystemCreateResponse), err } @@ -74,34 +69,34 @@ func (client filesystemClient) createPreparer(filesystem string, xMsProperties * if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - params.Set("resource", "filesystem") - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } // createResponder handles the response to the Create request. func (client filesystemClient) createResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK,http.StatusCreated) + err := validateResponse(resp, http.StatusOK, http.StatusCreated) if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &FilesystemCreateResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemCreateResponse{rawResponse: resp.Response()}, err } // Delete marks the filesystem for deletion. When a filesystem is deleted, a filesystem with the same identifier @@ -124,29 +119,27 @@ func (client filesystemClient) createResponder(resp pipeline.Response) (pipeline // completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is // required when using shared key authorization. func (client filesystemClient) Delete(ctx context.Context, filesystem string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemDeleteResponse, error) { - if err := validate([]validation{ - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.deletePreparer(filesystem, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*FilesystemDeleteResponse), err } @@ -156,37 +149,37 @@ func (client filesystemClient) deletePreparer(filesystem string, ifModifiedSince if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - params.Set("resource", "filesystem") - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } // deleteResponder handles the response to the Delete request. func (client filesystemClient) deleteResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK,http.StatusAccepted) + err := validateResponse(resp, http.StatusOK, http.StatusAccepted) if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &FilesystemDeleteResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemDeleteResponse{rawResponse: resp.Response()}, err } // GetProperties all system and user-defined filesystem properties are specified in the response headers. @@ -199,29 +192,27 @@ func (client filesystemClient) deleteResponder(resp pipeline.Response) (pipeline // fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is required when using // shared key authorization. func (client filesystemClient) GetProperties(ctx context.Context, filesystem string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemGetPropertiesResponse, error) { - if err := validate([]validation{ - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.getPropertiesPreparer(filesystem, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPropertiesResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*FilesystemGetPropertiesResponse), err } @@ -231,19 +222,19 @@ func (client filesystemClient) getPropertiesPreparer(filesystem string, xMsClien if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - params.Set("resource", "filesystem") - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -253,9 +244,9 @@ func (client filesystemClient) getPropertiesResponder(resp pipeline.Response) (p if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &FilesystemGetPropertiesResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemGetPropertiesResponse{rawResponse: resp.Response()}, err } // List list filesystems and their properties in given account. @@ -271,29 +262,26 @@ func (client filesystemClient) getPropertiesResponder(resp pipeline.Response) (p // completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is // required when using shared key authorization. func (client filesystemClient) List(ctx context.Context, prefix *string, continuation *string, maxResults *int32, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemList, error) { - if err := validate([]validation{ - { targetValue: maxResults, - constraints: []constraint{ {target: "maxResults", name: null, rule: false , - chain: []constraint{ {target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: maxResults, + constraints: []constraint{{target: "maxResults", name: null, rule: false, + chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.listPreparer(prefix, continuation, maxResults, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*FilesystemList), err } @@ -303,28 +291,28 @@ func (client filesystemClient) listPreparer(prefix *string, continuation *string if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - params.Set("resource", "account") - if prefix != nil && len(*prefix) > 0 { - params.Set("prefix", *prefix) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if maxResults != nil { - params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + params.Set("resource", "account") + if prefix != nil && len(*prefix) > 0 { + params.Set("prefix", *prefix) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if maxResults != nil { + params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -334,23 +322,23 @@ func (client filesystemClient) listResponder(resp pipeline.Response) (pipeline.R if resp == nil { return nil, err } - result:= &FilesystemList{rawResponse: resp.Response()} - if err != nil { - return result, err - } - defer resp.Response().Body.Close() - b, err:= ioutil.ReadAll(resp.Response().Body) - if err != nil { - return result, err - } - if len(b) > 0 { - b = removeBOM(b) - err = json.Unmarshal(b, result) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") - } - } - return result, nil + result := &FilesystemList{rawResponse: resp.Response()} + if err != nil { + return result, err + } + defer resp.Response().Body.Close() + b, err := ioutil.ReadAll(resp.Response().Body) + if err != nil { + return result, err + } + if len(b) > 0 { + b = removeBOM(b) + err = json.Unmarshal(b, result) + if err != nil { + return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") + } + } + return result, nil } // SetProperties set properties for the filesystem. This operation supports conditional HTTP requests. For more @@ -373,29 +361,27 @@ func (client filesystemClient) listResponder(resp pipeline.Response) (pipeline.R // timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated // Universal Time (UTC) for the request. This is required when using shared key authorization. func (client filesystemClient) SetProperties(ctx context.Context, filesystem string, xMsProperties *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*FilesystemSetPropertiesResponse, error) { - if err := validate([]validation{ - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.setPropertiesPreparer(filesystem, xMsProperties, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.setPropertiesResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*FilesystemSetPropertiesResponse), err } @@ -405,28 +391,28 @@ func (client filesystemClient) setPropertiesPreparer(filesystem string, xMsPrope if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - params.Set("resource", "filesystem") - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -436,8 +422,7 @@ func (client filesystemClient) setPropertiesResponder(resp pipeline.Response) (p if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &FilesystemSetPropertiesResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &FilesystemSetPropertiesResponse{rawResponse: resp.Response()}, err } - diff --git a/azbfs/zz_generated_path.go b/azbfs/zz_generated_path.go index ec743a731..270391e4c 100644 --- a/azbfs/zz_generated_path.go +++ b/azbfs/zz_generated_path.go @@ -4,29 +4,26 @@ package azbfs // Changes may cause incorrect behavior and will be lost if the code is regenerated. import ( - "net/url" - "github.com/Azure/azure-pipeline-go/pipeline" - "net/url" - "net/http" - "net/url" - "context" - "net/url" - "strconv" - "net/url" - "io" - "net/url" - "encoding/json" - "net/url" - "io/ioutil" + // begin manual edit to generated code + "context" + "encoding/json" + "github.com/Azure/azure-pipeline-go/pipeline" + "io" + "io/ioutil" + "net/http" + "net/url" + "strconv" + // end manual edit ) // pathClient is the azure Data Lake Storage provides storage for Hadoop and other big data workloads. type pathClient struct { - managementClient + managementClient } + // newPathClient creates an instance of the pathClient client. func newPathClient(url url.URL, p pipeline.Pipeline) pathClient { - return pathClient{newManagementClient(url, p)} + return pathClient{newManagementClient(url, p)} } // Create create or rename a file or directory. By default, the destination is overwritten and if the destination @@ -90,37 +87,33 @@ func newPathClient(url url.URL, p pipeline.Pipeline) pathClient { // operation completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. // This is required when using shared key authorization. func (client pathClient) Create(ctx context.Context, filesystem string, pathParameter string, resource PathResourceType, continuation *string, mode PathRenameModeType, cacheControl *string, contentEncoding *string, contentLanguage *string, contentDisposition *string, xMsCacheControl *string, xMsContentType *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentDisposition *string, xMsRenameSource *string, xMsLeaseID *string, xMsSourceLeaseID *string, xMsProperties *string, xMsPermissions *string, xMsUmask *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsSourceIfMatch *string, xMsSourceIfNoneMatch *string, xMsSourceIfModifiedSince *string, xMsSourceIfUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathCreateResponse, error) { - if err := validate([]validation{ - { targetValue: xMsLeaseID, - constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: xMsSourceLeaseID, - constraints: []constraint{ {target: "xMsSourceLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsSourceLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: xMsSourceLeaseID, + constraints: []constraint{{target: "xMsSourceLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsSourceLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.createPreparer(filesystem, pathParameter, resource, continuation, mode, cacheControl, contentEncoding, contentLanguage, contentDisposition, xMsCacheControl, xMsContentType, xMsContentEncoding, xMsContentLanguage, xMsContentDisposition, xMsRenameSource, xMsLeaseID, xMsSourceLeaseID, xMsProperties, xMsPermissions, xMsUmask, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsSourceIfMatch, xMsSourceIfNoneMatch, xMsSourceIfModifiedSince, xMsSourceIfUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.createResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*PathCreateResponse), err } @@ -130,108 +123,108 @@ func (client pathClient) createPreparer(filesystem string, pathParameter string, if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - if resource != PathResourceNone { - params.Set("resource", string(resource)) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if mode != PathRenameModeNone { - params.Set("mode", string(mode)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if cacheControl != nil { - req.Header.Set("Cache-Control", *cacheControl) - } - if contentEncoding != nil { - req.Header.Set("Content-Encoding", *contentEncoding) - } - if contentLanguage != nil { - req.Header.Set("Content-Language", *contentLanguage) - } - if contentDisposition != nil { - req.Header.Set("Content-Disposition", *contentDisposition) - } - if xMsCacheControl != nil { - req.Header.Set("x-ms-cache-control", *xMsCacheControl) - } - if xMsContentType != nil { - req.Header.Set("x-ms-content-type", *xMsContentType) - } - if xMsContentEncoding != nil { - req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) - } - if xMsContentLanguage != nil { - req.Header.Set("x-ms-content-language", *xMsContentLanguage) - } - if xMsContentDisposition != nil { - req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) - } - if xMsRenameSource != nil { - req.Header.Set("x-ms-rename-source", *xMsRenameSource) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsSourceLeaseID != nil { - req.Header.Set("x-ms-source-lease-id", *xMsSourceLeaseID) - } - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsPermissions != nil { - req.Header.Set("x-ms-permissions", *xMsPermissions) - } - if xMsUmask != nil { - req.Header.Set("x-ms-umask", *xMsUmask) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsSourceIfMatch != nil { - req.Header.Set("x-ms-source-if-match", *xMsSourceIfMatch) - } - if xMsSourceIfNoneMatch != nil { - req.Header.Set("x-ms-source-if-none-match", *xMsSourceIfNoneMatch) - } - if xMsSourceIfModifiedSince != nil { - req.Header.Set("x-ms-source-if-modified-since", *xMsSourceIfModifiedSince) - } - if xMsSourceIfUnmodifiedSince != nil { - req.Header.Set("x-ms-source-if-unmodified-since", *xMsSourceIfUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + if resource != PathResourceNone { + params.Set("resource", string(resource)) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if mode != PathRenameModeNone { + params.Set("mode", string(mode)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if cacheControl != nil { + req.Header.Set("Cache-Control", *cacheControl) + } + if contentEncoding != nil { + req.Header.Set("Content-Encoding", *contentEncoding) + } + if contentLanguage != nil { + req.Header.Set("Content-Language", *contentLanguage) + } + if contentDisposition != nil { + req.Header.Set("Content-Disposition", *contentDisposition) + } + if xMsCacheControl != nil { + req.Header.Set("x-ms-cache-control", *xMsCacheControl) + } + if xMsContentType != nil { + req.Header.Set("x-ms-content-type", *xMsContentType) + } + if xMsContentEncoding != nil { + req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) + } + if xMsContentLanguage != nil { + req.Header.Set("x-ms-content-language", *xMsContentLanguage) + } + if xMsContentDisposition != nil { + req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) + } + if xMsRenameSource != nil { + req.Header.Set("x-ms-rename-source", *xMsRenameSource) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsSourceLeaseID != nil { + req.Header.Set("x-ms-source-lease-id", *xMsSourceLeaseID) + } + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsPermissions != nil { + req.Header.Set("x-ms-permissions", *xMsPermissions) + } + if xMsUmask != nil { + req.Header.Set("x-ms-umask", *xMsUmask) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsSourceIfMatch != nil { + req.Header.Set("x-ms-source-if-match", *xMsSourceIfMatch) + } + if xMsSourceIfNoneMatch != nil { + req.Header.Set("x-ms-source-if-none-match", *xMsSourceIfNoneMatch) + } + if xMsSourceIfModifiedSince != nil { + req.Header.Set("x-ms-source-if-modified-since", *xMsSourceIfModifiedSince) + } + if xMsSourceIfUnmodifiedSince != nil { + req.Header.Set("x-ms-source-if-unmodified-since", *xMsSourceIfUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } // createResponder handles the response to the Create request. func (client pathClient) createResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK,http.StatusCreated) + err := validateResponse(resp, http.StatusOK, http.StatusCreated) if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &PathCreateResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathCreateResponse{rawResponse: resp.Response()}, err } // Delete delete the file or directory. This operation supports conditional HTTP requests. For more information, see @@ -256,33 +249,30 @@ func (client pathClient) createResponder(resp pipeline.Response) (pipeline.Respo // timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated // Universal Time (UTC) for the request. This is required when using shared key authorization. func (client pathClient) Delete(ctx context.Context, filesystem string, pathParameter string, recursive *bool, continuation *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathDeleteResponse, error) { - if err := validate([]validation{ - { targetValue: xMsLeaseID, - constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.deletePreparer(filesystem, pathParameter, recursive, continuation, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.deleteResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*PathDeleteResponse), err } @@ -292,39 +282,39 @@ func (client pathClient) deletePreparer(filesystem string, pathParameter string, if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - if recursive != nil { - params.Set("recursive", strconv.FormatBool(*recursive)) - } - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + if recursive != nil { + params.Set("recursive", strconv.FormatBool(*recursive)) + } + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -334,9 +324,9 @@ func (client pathClient) deleteResponder(resp pipeline.Response) (pipeline.Respo if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &PathDeleteResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathDeleteResponse{rawResponse: resp.Response()}, err } // GetProperties get Properties returns all system and user defined properties for a path. Get Status returns all @@ -367,33 +357,30 @@ func (client pathClient) deleteResponder(resp pipeline.Response) (pipeline.Respo // completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is // required when using shared key authorization. func (client pathClient) GetProperties(ctx context.Context, filesystem string, pathParameter string, action PathGetPropertiesActionType, upn *bool, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathGetPropertiesResponse, error) { - if err := validate([]validation{ - { targetValue: xMsLeaseID, - constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.getPropertiesPreparer(filesystem, pathParameter, action, upn, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.getPropertiesResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*PathGetPropertiesResponse), err } @@ -403,39 +390,39 @@ func (client pathClient) getPropertiesPreparer(filesystem string, pathParameter if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - if action != PathGetPropertiesActionNone { - params.Set("action", string(action)) - } - if upn != nil { - params.Set("upn", strconv.FormatBool(*upn)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + if action != PathGetPropertiesActionNone { + params.Set("action", string(action)) + } + if upn != nil { + params.Set("upn", strconv.FormatBool(*upn)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -445,9 +432,9 @@ func (client pathClient) getPropertiesResponder(resp pipeline.Response) (pipelin if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &PathGetPropertiesResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathGetPropertiesResponse{rawResponse: resp.Response()}, err } // Lease create and manage a lease to restrict write and delete access to the path. This operation supports conditional @@ -480,37 +467,33 @@ func (client pathClient) getPropertiesResponder(resp pipeline.Response) (pipelin // timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated // Universal Time (UTC) for the request. This is required when using shared key authorization. func (client pathClient) Lease(ctx context.Context, xMsLeaseAction PathLeaseActionType, filesystem string, pathParameter string, xMsLeaseDuration *int32, xMsLeaseBreakPeriod *int32, xMsLeaseID *string, xMsProposedLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathLeaseResponse, error) { - if err := validate([]validation{ - { targetValue: xMsLeaseID, - constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: xMsProposedLeaseID, - constraints: []constraint{ {target: "xMsProposedLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: xMsProposedLeaseID, + constraints: []constraint{{target: "xMsProposedLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsProposedLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.leasePreparer(xMsLeaseAction, filesystem, pathParameter, xMsLeaseDuration, xMsLeaseBreakPeriod, xMsLeaseID, xMsProposedLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.leaseResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*PathLeaseResponse), err } @@ -520,55 +503,55 @@ func (client pathClient) leasePreparer(xMsLeaseAction PathLeaseActionType, files if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - req.Header.Set("x-ms-lease-action", string(xMsLeaseAction)) - if xMsLeaseDuration != nil { - req.Header.Set("x-ms-lease-duration", strconv.FormatInt(int64(*xMsLeaseDuration), 10)) - } - if xMsLeaseBreakPeriod != nil { - req.Header.Set("x-ms-lease-break-period", strconv.FormatInt(int64(*xMsLeaseBreakPeriod), 10)) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsProposedLeaseID != nil { - req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + req.Header.Set("x-ms-lease-action", string(xMsLeaseAction)) + if xMsLeaseDuration != nil { + req.Header.Set("x-ms-lease-duration", strconv.FormatInt(int64(*xMsLeaseDuration), 10)) + } + if xMsLeaseBreakPeriod != nil { + req.Header.Set("x-ms-lease-break-period", strconv.FormatInt(int64(*xMsLeaseBreakPeriod), 10)) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsProposedLeaseID != nil { + req.Header.Set("x-ms-proposed-lease-id", *xMsProposedLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } // leaseResponder handles the response to the Lease request. func (client pathClient) leaseResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK,http.StatusCreated,http.StatusAccepted) + err := validateResponse(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &PathLeaseResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathLeaseResponse{rawResponse: resp.Response()}, err } // List list filesystem paths and their properties. @@ -593,33 +576,30 @@ func (client pathClient) leaseResponder(resp pipeline.Response) (pipeline.Respon // is specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key // authorization. func (client pathClient) List(ctx context.Context, recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathList, error) { - if err := validate([]validation{ - { targetValue: maxResults, - constraints: []constraint{ {target: "maxResults", name: null, rule: false , - chain: []constraint{ {target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}, - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: maxResults, + constraints: []constraint{{target: "maxResults", name: null, rule: false, + chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.listPreparer(recursive, filesystem, directory, continuation, maxResults, upn, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*PathList), err } @@ -629,32 +609,32 @@ func (client pathClient) listPreparer(recursive bool, filesystem string, directo if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - if directory != nil && len(*directory) > 0 { - params.Set("directory", *directory) - } - params.Set("recursive", strconv.FormatBool(recursive)) - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if maxResults != nil { - params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) - } - if upn != nil { - params.Set("upn", strconv.FormatBool(*upn)) - } - params.Set("resource", "filesystem") - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + if directory != nil && len(*directory) > 0 { + params.Set("directory", *directory) + } + params.Set("recursive", strconv.FormatBool(recursive)) + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if maxResults != nil { + params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) + } + if upn != nil { + params.Set("upn", strconv.FormatBool(*upn)) + } + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } @@ -664,23 +644,23 @@ func (client pathClient) listResponder(resp pipeline.Response) (pipeline.Respons if resp == nil { return nil, err } - result:= &PathList{rawResponse: resp.Response()} - if err != nil { - return result, err - } - defer resp.Response().Body.Close() - b, err:= ioutil.ReadAll(resp.Response().Body) - if err != nil { - return result, err - } - if len(b) > 0 { - b = removeBOM(b) - err = json.Unmarshal(b, result) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") - } - } - return result, nil + result := &PathList{rawResponse: resp.Response()} + if err != nil { + return result, err + } + defer resp.Response().Body.Close() + b, err := ioutil.ReadAll(resp.Response().Body) + if err != nil { + return result, err + } + if len(b) > 0 { + b = removeBOM(b) + err = json.Unmarshal(b, result) + if err != nil { + return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") + } + } + return result, nil } // Read read the contents of a file. For read operations, range requests are supported. This operation supports @@ -703,33 +683,30 @@ func (client pathClient) listResponder(resp pipeline.Response) (pipeline.Respons // completes, the operation fails. xMsDate is specifies the Coordinated Universal Time (UTC) for the request. This is // required when using shared key authorization. func (client pathClient) Read(ctx context.Context, filesystem string, pathParameter string, rangeParameter *string, xMsLeaseID *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*ReadResponse, error) { - if err := validate([]validation{ - { targetValue: xMsLeaseID, - constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.readPreparer(filesystem, pathParameter, rangeParameter, xMsLeaseID, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.readResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*ReadResponse), err } @@ -739,46 +716,46 @@ func (client pathClient) readPreparer(filesystem string, pathParameter string, r if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if rangeParameter != nil { - req.Header.Set("Range", *rangeParameter) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if rangeParameter != nil { + req.Header.Set("Range", *rangeParameter) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } // readResponder handles the response to the Read request. func (client pathClient) readResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK,http.StatusPartialContent) + err := validateResponse(resp, http.StatusOK, http.StatusPartialContent) if resp == nil { return nil, err } - return &ReadResponse{rawResponse: resp.Response()}, err + return &ReadResponse{rawResponse: resp.Response()}, err } // Update uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties @@ -867,37 +844,33 @@ func (client pathClient) readResponder(resp pipeline.Response) (pipeline.Respons // timeout value elapses before the operation completes, the operation fails. xMsDate is specifies the Coordinated // Universal Time (UTC) for the request. This is required when using shared key authorization. func (client pathClient) Update(ctx context.Context, action PathUpdateActionType, filesystem string, pathParameter string, position *int64, retainUncommittedData *bool, closeParameter *bool, contentLength *int64, xMsLeaseID *string, xMsCacheControl *string, xMsContentType *string, xMsContentDisposition *string, xMsContentEncoding *string, xMsContentLanguage *string, xMsContentMd5 *string, xMsProperties *string, xMsOwner *string, xMsGroup *string, xMsPermissions *string, xMsACL *string, ifMatch *string, ifNoneMatch *string, ifModifiedSince *string, ifUnmodifiedSince *string, xHTTPMethodOverride *string, body io.ReadSeeker, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathUpdateResponse, error) { - if err := validate([]validation{ - { targetValue: contentLength, - constraints: []constraint{ {target: "contentLength", name: null, rule: false , - chain: []constraint{ {target: "contentLength", name: inclusiveMinimum, rule: 0, chain: nil }, - }}}}, - { targetValue: xMsLeaseID, - constraints: []constraint{ {target: "xMsLeaseID", name: null, rule: false , - chain: []constraint{ {target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: filesystem, - constraints: []constraint{ {target: "filesystem", name: maxLength, rule: 63, chain: nil }, - {target: "filesystem", name: minLength, rule: 3, chain: nil }, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil }}}, - { targetValue: xMsClientRequestID, - constraints: []constraint{ {target: "xMsClientRequestID", name: null, rule: false , - chain: []constraint{ {target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil }, - }}}}, - { targetValue: timeout, - constraints: []constraint{ {target: "timeout", name: null, rule: false , - chain: []constraint{ {target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil }, - }}}}}); err != nil { - return nil, err - } + if err := validate([]validation{ + {targetValue: contentLength, + constraints: []constraint{{target: "contentLength", name: null, rule: false, + chain: []constraint{{target: "contentLength", name: inclusiveMinimum, rule: 0, chain: nil}}}}}, + {targetValue: xMsLeaseID, + constraints: []constraint{{target: "xMsLeaseID", name: null, rule: false, + chain: []constraint{{target: "xMsLeaseID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } req, err := client.updatePreparer(action, filesystem, pathParameter, position, retainUncommittedData, closeParameter, contentLength, xMsLeaseID, xMsCacheControl, xMsContentType, xMsContentDisposition, xMsContentEncoding, xMsContentLanguage, xMsContentMd5, xMsProperties, xMsOwner, xMsGroup, xMsPermissions, xMsACL, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, xHTTPMethodOverride, body, xMsClientRequestID, timeout, xMsDate) if err != nil { return nil, err } resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.updateResponder}, req) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } return resp.(*PathUpdateResponse), err } @@ -914,93 +887,92 @@ func (client pathClient) updatePreparer(action PathUpdateActionType, filesystem if err != nil { return req, pipeline.NewError(err, "failed to create request") } - params := req.URL.Query() - params.Set("action", string(action)) - if position != nil { - params.Set("position", strconv.FormatInt(*position, 10)) - } - if retainUncommittedData != nil { - params.Set("retainUncommittedData", strconv.FormatBool(*retainUncommittedData)) - } - if closeParameter != nil { - params.Set("close", strconv.FormatBool(*closeParameter)) - } - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if contentLength != nil { - req.Header.Set("Content-Length", strconv.FormatInt(*contentLength, 10)) - } - if xMsLeaseID != nil { - req.Header.Set("x-ms-lease-id", *xMsLeaseID) - } - if xMsCacheControl != nil { - req.Header.Set("x-ms-cache-control", *xMsCacheControl) - } - if xMsContentType != nil { - req.Header.Set("x-ms-content-type", *xMsContentType) - } - if xMsContentDisposition != nil { - req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) - } - if xMsContentEncoding != nil { - req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) - } - if xMsContentLanguage != nil { - req.Header.Set("x-ms-content-language", *xMsContentLanguage) - } - if xMsContentMd5 != nil { - req.Header.Set("x-ms-content-md5", *xMsContentMd5) - } - if xMsProperties != nil { - req.Header.Set("x-ms-properties", *xMsProperties) - } - if xMsOwner != nil { - req.Header.Set("x-ms-owner", *xMsOwner) - } - if xMsGroup != nil { - req.Header.Set("x-ms-group", *xMsGroup) - } - if xMsPermissions != nil { - req.Header.Set("x-ms-permissions", *xMsPermissions) - } - if xMsACL != nil { - req.Header.Set("x-ms-acl", *xMsACL) - } - if ifMatch != nil { - req.Header.Set("If-Match", *ifMatch) - } - if ifNoneMatch != nil { - req.Header.Set("If-None-Match", *ifNoneMatch) - } - if ifModifiedSince != nil { - req.Header.Set("If-Modified-Since", *ifModifiedSince) - } - if ifUnmodifiedSince != nil { - req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) - } - if xHTTPMethodOverride != nil { - req.Header.Set("x-http-method-override", *xHTTPMethodOverride) - } - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) + params := req.URL.Query() + params.Set("action", string(action)) + if position != nil { + params.Set("position", strconv.FormatInt(*position, 10)) + } + if retainUncommittedData != nil { + params.Set("retainUncommittedData", strconv.FormatBool(*retainUncommittedData)) + } + if closeParameter != nil { + params.Set("close", strconv.FormatBool(*closeParameter)) + } + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if contentLength != nil { + req.Header.Set("Content-Length", strconv.FormatInt(*contentLength, 10)) + } + if xMsLeaseID != nil { + req.Header.Set("x-ms-lease-id", *xMsLeaseID) + } + if xMsCacheControl != nil { + req.Header.Set("x-ms-cache-control", *xMsCacheControl) + } + if xMsContentType != nil { + req.Header.Set("x-ms-content-type", *xMsContentType) + } + if xMsContentDisposition != nil { + req.Header.Set("x-ms-content-disposition", *xMsContentDisposition) + } + if xMsContentEncoding != nil { + req.Header.Set("x-ms-content-encoding", *xMsContentEncoding) + } + if xMsContentLanguage != nil { + req.Header.Set("x-ms-content-language", *xMsContentLanguage) + } + if xMsContentMd5 != nil { + req.Header.Set("x-ms-content-md5", *xMsContentMd5) + } + if xMsProperties != nil { + req.Header.Set("x-ms-properties", *xMsProperties) + } + if xMsOwner != nil { + req.Header.Set("x-ms-owner", *xMsOwner) + } + if xMsGroup != nil { + req.Header.Set("x-ms-group", *xMsGroup) + } + if xMsPermissions != nil { + req.Header.Set("x-ms-permissions", *xMsPermissions) + } + if xMsACL != nil { + req.Header.Set("x-ms-acl", *xMsACL) + } + if ifMatch != nil { + req.Header.Set("If-Match", *ifMatch) + } + if ifNoneMatch != nil { + req.Header.Set("If-None-Match", *ifNoneMatch) + } + if ifModifiedSince != nil { + req.Header.Set("If-Modified-Since", *ifModifiedSince) + } + if ifUnmodifiedSince != nil { + req.Header.Set("If-Unmodified-Since", *ifUnmodifiedSince) + } + if xHTTPMethodOverride != nil { + req.Header.Set("x-http-method-override", *xHTTPMethodOverride) + } + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) return req, nil } // updateResponder handles the response to the Update request. func (client pathClient) updateResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK,http.StatusAccepted) + err := validateResponse(resp, http.StatusOK, http.StatusAccepted) if resp == nil { return nil, err } - io.Copy(ioutil.Discard, resp.Response().Body) - resp.Response().Body.Close() - return &PathUpdateResponse{rawResponse: resp.Response()}, err + io.Copy(ioutil.Discard, resp.Response().Body) + resp.Response().Body.Close() + return &PathUpdateResponse{rawResponse: resp.Response()}, err } - From 971159231ea914ceb77e2a78762b8097a6cf6361 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 12 Feb 2019 20:15:41 +1300 Subject: [PATCH 24/64] Manually tweak generated file to let JSON bool be string-quoted Need to find out if there's a better way to do this. Interestingly, there's only one bool in the JSON models. --- azbfs/zz_generated_models.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/azbfs/zz_generated_models.go b/azbfs/zz_generated_models.go index 734eb9f61..4c77b9dae 100644 --- a/azbfs/zz_generated_models.go +++ b/azbfs/zz_generated_models.go @@ -379,8 +379,12 @@ func (fspr FilesystemSetPropertiesResponse) XMsVersion() string { // Path ... type Path struct { - Name *string `json:"name,omitempty"` - IsDirectory *bool `json:"isDirectory,omitempty"` + Name *string `json:"name,omitempty"` + + // begin manual edit to generated code + IsDirectory *bool `json:"isDirectory,string,omitempty"` + // end manual edit + LastModified *string `json:"lastModified,omitempty"` ETag *string `json:"eTag,omitempty"` ContentLength *int64 `json:"contentLength,omitempty"` From 21b6a83c058e7715c2ba495a88da52289b5e6d8b Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Wed, 13 Feb 2019 14:09:34 +1300 Subject: [PATCH 25/64] Show blobFS enumeration errors --- cmd/copyDownloadBlobFSEnumerator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/copyDownloadBlobFSEnumerator.go b/cmd/copyDownloadBlobFSEnumerator.go index ab86af77f..3809ab626 100644 --- a/cmd/copyDownloadBlobFSEnumerator.go +++ b/cmd/copyDownloadBlobFSEnumerator.go @@ -174,7 +174,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { for { dListResp, err := directoryURL.ListDirectorySegment(ctx, &continuationMarker, true) if err != nil { - return fmt.Errorf("error listing the files inside the given source url %s", directoryURL.String()) + return fmt.Errorf("error listing the files inside the given source url %s: %s", directoryURL.String(), err.Error()) } // get only the files inside the given path From c88797e074ccd95aa410895b81bc259c34520765 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Wed, 13 Feb 2019 14:14:05 +1300 Subject: [PATCH 26/64] Allow content length to be string quoted in JSON replies Since Go won't parse it inside quotes if we don't tell Go in advance that quotes may surround it --- azbfs/zz_generated_models.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/azbfs/zz_generated_models.go b/azbfs/zz_generated_models.go index 4c77b9dae..96bbdacd3 100644 --- a/azbfs/zz_generated_models.go +++ b/azbfs/zz_generated_models.go @@ -385,12 +385,16 @@ type Path struct { IsDirectory *bool `json:"isDirectory,string,omitempty"` // end manual edit - LastModified *string `json:"lastModified,omitempty"` - ETag *string `json:"eTag,omitempty"` - ContentLength *int64 `json:"contentLength,omitempty"` - Owner *string `json:"owner,omitempty"` - Group *string `json:"group,omitempty"` - Permissions *string `json:"permissions,omitempty"` + LastModified *string `json:"lastModified,omitempty"` + ETag *string `json:"eTag,omitempty"` + + // begin manual edit to generated code + ContentLength *int64 `json:"contentLength,string,omitempty"` + // end manual edit + + Owner *string `json:"owner,omitempty"` + Group *string `json:"group,omitempty"` + Permissions *string `json:"permissions,omitempty"` } // PathCreateResponse ... From 89cf6b1e40b689e8c05a819d9c503c6ce1e2878d Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Wed, 13 Feb 2019 15:42:35 +1300 Subject: [PATCH 27/64] Allow complete skipping of computation of MD5 hashes on download To accomodate those users who chose to prioritize CPU usage over integrity checking. Also, may be useful for performance in cases where a particular remote back end requires an extra round trip to get the MD5 data. --- common/chunkedFileWriter.go | 10 ++++++++- common/fe-ste-models.go | 13 ++++++----- common/nullHasher.go | 45 +++++++++++++++++++++++++++++++++++++ ste/md5Comparer.go | 4 ++++ ste/xfer-remoteToLocal.go | 3 ++- 5 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 common/nullHasher.go diff --git a/common/chunkedFileWriter.go b/common/chunkedFileWriter.go index 869dd808b..d048c194d 100644 --- a/common/chunkedFileWriter.go +++ b/common/chunkedFileWriter.go @@ -86,6 +86,9 @@ type chunkedFileWriter struct { // controls body-read retries. Public so value can be shared with retryReader maxRetryPerDownloadBody int + + // how will hashes be validated? + md5ValidationOption HashValidationOption } type fileChunk struct { @@ -93,7 +96,7 @@ type fileChunk struct { data []byte } -func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheLimiter CacheLimiter, chunkLogger ChunkStatusLogger, file io.WriteCloser, numChunks uint32, maxBodyRetries int) ChunkedFileWriter { +func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheLimiter CacheLimiter, chunkLogger ChunkStatusLogger, file io.WriteCloser, numChunks uint32, maxBodyRetries int, md5ValidationOption HashValidationOption) ChunkedFileWriter { // Set max size for buffered channel. The upper limit here is believed to be generous, given worker routine drains it constantly. // Use num chunks in file if lower than the upper limit, to prevent allocating RAM for lots of large channel buffers when dealing with // very large numbers of very small files. @@ -109,6 +112,7 @@ func NewChunkedFileWriter(ctx context.Context, slicePool ByteSlicePooler, cacheL newUnorderedChunks: make(chan fileChunk, chanBufferSize), creationTime: time.Now(), maxRetryPerDownloadBody: maxBodyRetries, + md5ValidationOption: md5ValidationOption, } go w.workerRoutine(ctx) return w @@ -203,6 +207,10 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { nextOffsetToSave := int64(0) unsavedChunksByFileOffset := make(map[int64]fileChunk) md5Hasher := md5.New() + if w.md5ValidationOption == EHashValidationOption.NoCheck() { + // save CPU time by not even computing a hash, if we are not going to check it + md5Hasher = &nullHasher{} + } for { var newChunk fileChunk diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index b4ac37c40..cc751af3a 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -558,18 +558,21 @@ var DefaultHashValidationOption = EHashValidationOption.FailIfDifferent() type HashValidationOption uint8 -// LogOnly is the least strict option -func (HashValidationOption) LogOnly() HashValidationOption { return HashValidationOption(0) } - // FailIfDifferent says fail if hashes different, but NOT fail if saved hash is // totally missing. This is a balance of convenience (for cases where no hash is saved) vs strictness // (to validate strictly when one is present) -func (HashValidationOption) FailIfDifferent() HashValidationOption { return HashValidationOption(1) } +func (HashValidationOption) FailIfDifferent() HashValidationOption { return HashValidationOption(0) } + +// Do not check hashes at download time at all +func (HashValidationOption) NoCheck() HashValidationOption { return HashValidationOption(1) } + +// LogOnly means only log if missing or different, don't fail the transfer +func (HashValidationOption) LogOnly() HashValidationOption { return HashValidationOption(2) } // FailIfDifferentOrMissing is the strictest option, and useful for testing or validation in cases when // we _know_ there should be a hash func (HashValidationOption) FailIfDifferentOrMissing() HashValidationOption { - return HashValidationOption(2) + return HashValidationOption(3) } func (hvo HashValidationOption) String() string { diff --git a/common/nullHasher.go b/common/nullHasher.go new file mode 100644 index 000000000..a5a943d4b --- /dev/null +++ b/common/nullHasher.go @@ -0,0 +1,45 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package common + +// A hash.Hash implementation that does nothing +type nullHasher struct{} + +func (*nullHasher) Write(p []byte) (n int, err error) { + // noop + return 0, nil +} + +func (*nullHasher) Sum(b []byte) []byte { + return make([]byte, 0) +} + +func (*nullHasher) Reset() { + // noop +} + +func (*nullHasher) Size() int { + return 0 +} + +func (*nullHasher) BlockSize() int { + return 1 +} diff --git a/ste/md5Comparer.go b/ste/md5Comparer.go index e7c0ae45a..fab8a1ee5 100644 --- a/ste/md5Comparer.go +++ b/ste/md5Comparer.go @@ -54,6 +54,10 @@ var errActualMd5NotComputed = errors.New("no MDB was computed within this applic // is respond to non-nil errors func (c *md5Comparer) Check() error { + if c.validationOption == common.EHashValidationOption.NoCheck() { + return nil + } + if c.actualAsSaved == nil || len(c.actualAsSaved) == 0 { return errActualMd5NotComputed // Should never happen, so there's no way to opt out of this error being returned if it DOES happen } diff --git a/ste/xfer-remoteToLocal.go b/ste/xfer-remoteToLocal.go index b30991ccd..a3ffcefdd 100644 --- a/ste/xfer-remoteToLocal.go +++ b/ste/xfer-remoteToLocal.go @@ -118,7 +118,8 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, chunkLogger, dstFile, numChunks, - MaxRetryPerDownloadBody) + MaxRetryPerDownloadBody, + jptm.MD5ValidationOption()) // step 5c: tell jptm what to expect, and how to clean up at the end jptm.SetNumberOfChunks(numChunks) From 61d6013fa5416321cbc56172022edba6936a387e Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Wed, 13 Feb 2019 18:01:01 +1300 Subject: [PATCH 28/64] Modify generated blobFS code to support MD5s --- azbfs/zz_generated_models.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/azbfs/zz_generated_models.go b/azbfs/zz_generated_models.go index 96bbdacd3..4fb458266 100644 --- a/azbfs/zz_generated_models.go +++ b/azbfs/zz_generated_models.go @@ -4,6 +4,7 @@ package azbfs // Changes may cause incorrect behavior and will be lost if the code is regenerated. import ( + "encoding/base64" "io" "net/http" "reflect" @@ -392,6 +393,15 @@ type Path struct { ContentLength *int64 `json:"contentLength,string,omitempty"` // end manual edit + // begin manual addition to generated code + // TODO: + // (a) How can we verify this will actually work with the JSON that the service will emit, when the service starts to do so? + // (b) One day, consider converting this to use a custom type, that implements TextMarshaller, as has been done + // for the XML-based responses in other SDKs. For now, the decoding from Base64 is up to the caller, and the name is chosen + // to reflect that. + ContentMD5Base64 *string `json:"contentMd5,string,omitempty"` + // end manual addition + Owner *string `json:"owner,omitempty"` Group *string `json:"group,omitempty"` Permissions *string `json:"permissions,omitempty"` @@ -559,10 +569,23 @@ func (pgpr PathGetPropertiesResponse) ContentLength() int64 { } // ContentMD5 returns the value for header Content-MD5. -func (pgpr PathGetPropertiesResponse) ContentMD5() string { - return pgpr.rawResponse.Header.Get("Content-MD5") +// begin manual edit to generated code +func (pgpr PathGetPropertiesResponse) ContentMD5() []byte { + // TODO: why did I have to generate this myself, whereas for blob API corresponding function seems to be + // auto-generated from the Swagger? + s := pgpr.rawResponse.Header.Get("Content-MD5") + if s == "" { + return nil + } + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + b = nil + } + return b } +// end manual edit to generated code + // ContentRange returns the value for header Content-Range. func (pgpr PathGetPropertiesResponse) ContentRange() string { return pgpr.rawResponse.Header.Get("Content-Range") From 7d8d5fbc5e6e628e25fbb7bb0762107b05aff330 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Wed, 13 Feb 2019 18:01:27 +1300 Subject: [PATCH 29/64] Add MD5 support to our wrappers around the BlobFS generated code --- azbfs/url_file.go | 12 ++++++++++-- azbfs/zz_response_model.go | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/azbfs/url_file.go b/azbfs/url_file.go index 74818c42b..515524215 100644 --- a/azbfs/url_file.go +++ b/azbfs/url_file.go @@ -2,6 +2,7 @@ package azbfs import ( "context" + "encoding/base64" "net/url" "github.com/Azure/azure-pipeline-go/pipeline" @@ -141,7 +142,8 @@ func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeke } // flushes writes previously uploaded data to a file -func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*PathUpdateResponse, error) { +// The contentMd5 parameter, if not nil, should represent the MD5 hash that has been computed for the file as whole +func (f FileURL) FlushData(ctx context.Context, fileSize int64, contentMd5 []byte) (*PathUpdateResponse, error) { if fileSize < 0 { panic("fileSize must be >= 0") } @@ -150,6 +152,12 @@ func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*PathUpdateResp // azcopy does not need this retainUncommittedData := false + var md5InBase64 *string = nil + if len(contentMd5) > 0 { + enc := base64.StdEncoding.EncodeToString(contentMd5) + md5InBase64 = &enc + } + // TODO: the go http client has a problem with PATCH and content-length header // we should investigate and report the issue overrideHttpVerb := "PATCH" @@ -160,7 +168,7 @@ func (f FileURL) FlushData(ctx context.Context, fileSize int64) (*PathUpdateResp // TransactionalContentMD5 isn't supported currently. return f.fileClient.Update(ctx, PathUpdateActionFlush, f.fileSystemName, f.path, &fileSize, &retainUncommittedData, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, + nil, nil, nil, md5InBase64, nil, nil, nil, nil, nil, nil, nil, nil, nil, &overrideHttpVerb, nil, nil, nil, nil) } diff --git a/azbfs/zz_response_model.go b/azbfs/zz_response_model.go index ea9c5281c..84ab04701 100644 --- a/azbfs/zz_response_model.go +++ b/azbfs/zz_response_model.go @@ -210,6 +210,11 @@ func (dgpr DirectoryGetPropertiesResponse) XMsVersion() string { return PathGetPropertiesResponse(dgpr).XMsVersion() } +// ContentMD5 returns the value for header Content-MD5. +func (dgpr DirectoryGetPropertiesResponse) ContentMD5() []byte { + return PathGetPropertiesResponse(dgpr).ContentMD5() +} + // DirectoryListResponse is the ListSchema response type. This type declaration is used to implement useful methods on // ListPath response type DirectoryListResponse PathList // TODO: Used to by ListPathResponse. Have I changed it to the right thing? From 9f17f3a3af7a55597822ebc37d49db3286f13c0d Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Wed, 13 Feb 2019 18:01:46 +1300 Subject: [PATCH 30/64] Set and check MD5s for BlobFS --- cmd/copyDownloadBlobFSEnumerator.go | 35 ++++++++++++++++++++++++++++- ste/uploader-blobFS.go | 12 +++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/cmd/copyDownloadBlobFSEnumerator.go b/cmd/copyDownloadBlobFSEnumerator.go index 3809ab626..6c926cf36 100644 --- a/cmd/copyDownloadBlobFSEnumerator.go +++ b/cmd/copyDownloadBlobFSEnumerator.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "encoding/base64" "errors" "fmt" "net/url" @@ -61,6 +62,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: destination, LastModifiedTime: e.parseLmt(props.LastModified()), SourceSize: fileSize, + ContentMD5: props.ContentMD5(), }, cca) return e.dispatchFinalPart(cca) @@ -111,7 +113,9 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Source: srcURL.String(), Destination: util.generateLocalPath(cca.destination, fileRelativePath), LastModifiedTime: e.parseLmt(fileProperties.LastModified()), - SourceSize: fileSize}, cca) + SourceSize: fileSize, + ContentMD5: fileProperties.ContentMD5(), + }, cca) continue } @@ -139,6 +143,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: util.generateLocalPath(cca.destination, relativePath), LastModifiedTime: e.parseLmt(*fileItem.LastModified), SourceSize: *fileItem.ContentLength, + ContentMD5: getContentMd5(ctx, directoryURL, fileItem, cca.md5ValidationOption), }, cca) }, ) @@ -186,6 +191,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { Destination: util.generateLocalPath(cca.destination, util.getRelativePath(fsUrlParts.DirectoryOrFilePath, *path.Name)), LastModifiedTime: e.parseLmt(*path.LastModified), SourceSize: *path.ContentLength, + ContentMD5: getContentMd5(ctx, directoryURL, path, cca.md5ValidationOption), }, cca) } @@ -206,6 +212,33 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { return nil } +func getContentMd5(ctx context.Context, directoryURL azbfs.DirectoryURL, file azbfs.Path, md5ValidationOption common.HashValidationOption) []byte { + if md5ValidationOption == common.EHashValidationOption.NoCheck() { + return nil // not gonna check it, so don't need it + } + + var returnValueForError []byte = nil // If we get an error, we just act like there was no content MD5. If validation is set to fail on error, this will fail the transfer of this file later on (at the time of the MD5 check) + + // convert format of what we have, if we have something + if file.ContentMD5Base64 != nil { + value, err := base64.StdEncoding.DecodeString(*file.ContentMD5Base64) + if err != nil { + return returnValueForError + } + return value + } + + // Fall back to making a new round trip to the server + // This is an interim measure, so that we can still validate MD5s even before they are being returned in the server's + // PathList response + fileURL := directoryURL.FileSystemURL().NewDirectoryURL(*file.Name) + props, err := fileURL.GetProperties(ctx) + if err != nil { + return returnValueForError + } + return props.ContentMD5() +} + func (e *copyDownloadBlobFSEnumerator) parseLmt(lastModifiedTime string) time.Time { // if last modified time is available, parse it // otherwise use the current time as last modified time diff --git a/ste/uploader-blobFS.go b/ste/uploader-blobFS.go index 69c0fe624..8800a1a1a 100644 --- a/ste/uploader-blobFS.go +++ b/ste/uploader-blobFS.go @@ -168,9 +168,15 @@ func (u *blobFSUploader) Epilogue() { // flush if jptm.TransferStatus() > 0 { - _, err := u.fileURL.FlushData(jptm.Context(), jptm.Info().SourceSize) - if err != nil { - jptm.FailActiveUpload("Flushing data", err) + md5Hash, ok := <-u.md5Channel + if ok { + _, err := u.fileURL.FlushData(jptm.Context(), jptm.Info().SourceSize, md5Hash) + if err != nil { + jptm.FailActiveUpload("Flushing data", err) + // don't return, since need cleanup below + } + } else { + jptm.FailActiveUpload("Getting hash", errNoHash) // don't return, since need cleanup below } } From a957457c7b5e20467c13f349d8020e140e0b7087 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Fri, 15 Feb 2019 11:46:06 +1300 Subject: [PATCH 31/64] Fix action type on GetProperties so that we get the MD5s from BlobFS --- azbfs/url_directory.go | 6 +++++- azbfs/url_file.go | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/azbfs/url_directory.go b/azbfs/url_directory.go index 3ccf34b2b..df7048acd 100644 --- a/azbfs/url_directory.go +++ b/azbfs/url_directory.go @@ -84,7 +84,11 @@ func (d DirectoryURL) Delete(ctx context.Context, continuationString *string, re // GetProperties returns the directory's metadata and system properties. func (d DirectoryURL) GetProperties(ctx context.Context) (*DirectoryGetPropertiesResponse, error) { - resp, err := d.directoryClient.GetProperties(ctx, d.filesystem, d.pathParameter, PathGetPropertiesActionGetStatus, nil, nil, + // Action MUST be "none", not "getStatus" because the latter does not include the MD5, and + // sometimes we call this method on things that are actually files + action := PathGetPropertiesActionNone + + resp, err := d.directoryClient.GetProperties(ctx, d.filesystem, d.pathParameter, action, nil, nil, nil, nil, nil, nil, nil, nil, nil) return (*DirectoryGetPropertiesResponse)(resp), err } diff --git a/azbfs/url_file.go b/azbfs/url_file.go index 515524215..62e7ecd6c 100644 --- a/azbfs/url_file.go +++ b/azbfs/url_file.go @@ -110,7 +110,11 @@ func (f FileURL) Delete(ctx context.Context) (*PathDeleteResponse, error) { // GetProperties returns the file's metadata and properties. // For more information, see https://docs.microsoft.com/rest/api/storageservices/get-file-properties. func (f FileURL) GetProperties(ctx context.Context) (*PathGetPropertiesResponse, error) { - return f.fileClient.GetProperties(ctx, f.fileSystemName, f.path, PathGetPropertiesActionGetStatus, nil, + // Action MUST be "none", not "getStatus" because the latter does not include the MD5, and + // sometimes we call this method on things that are actually files + action := PathGetPropertiesActionNone + + return f.fileClient.GetProperties(ctx, f.fileSystemName, f.path, action, nil, nil, nil, nil, nil, nil, nil, nil, nil) } From eb449c21a51441aeda71943f835af6248b33f660 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Fri, 15 Feb 2019 11:53:24 +1300 Subject: [PATCH 32/64] Update comments on the need to send PATCH as PUT to work around Go SDK behaviour --- azbfs/url_file.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/azbfs/url_file.go b/azbfs/url_file.go index 62e7ecd6c..3c9e79614 100644 --- a/azbfs/url_file.go +++ b/azbfs/url_file.go @@ -136,6 +136,14 @@ func (f FileURL) AppendData(ctx context.Context, offset int64, body io.ReadSeeke // TODO: the go http client has a problem with PATCH and content-length header // we should investigate and report the issue + // Note: the "offending" code in the Go SDK is: func (t *transferWriter) shouldSendContentLength() bool + // That code suggests that a workaround would be to specify a Transfer-Encoding of "identity", + // but we haven't yet found any way to actually set that header, so that workaround does't + // seem to work. (Just setting Transfer-Encoding like a normal header doesn't seem to work.) + // Looks like it might actually be impossible to set + // the Transfer-Encoding header, because bradfitz wrote: "as a general rule of thumb, you don't get to mess + // with [that field] too much. The net/http package owns much of its behavior." + // https://grokbase.com/t/gg/golang-nuts/15bg66ryd9/go-nuts-cant-write-encoding-other-than-chunked-in-the-transfer-encoding-field-of-http-request overrideHttpVerb := "PATCH" // TransactionalContentMD5 isn't supported currently. @@ -164,6 +172,7 @@ func (f FileURL) FlushData(ctx context.Context, fileSize int64, contentMd5 []byt // TODO: the go http client has a problem with PATCH and content-length header // we should investigate and report the issue + // See similar todo, with larger comments, in AppendData overrideHttpVerb := "PATCH" // TODO: feb 2019 API update: review the use of closeParameter here. Should it be true? From fe0d47fe58147082fbd17714bc88d56a6198df70 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Fri, 15 Feb 2019 12:06:07 +1300 Subject: [PATCH 33/64] Update comment and TODO --- cmd/copyDownloadBlobFSEnumerator.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/copyDownloadBlobFSEnumerator.go b/cmd/copyDownloadBlobFSEnumerator.go index 6c926cf36..8c4814147 100644 --- a/cmd/copyDownloadBlobFSEnumerator.go +++ b/cmd/copyDownloadBlobFSEnumerator.go @@ -219,7 +219,7 @@ func getContentMd5(ctx context.Context, directoryURL azbfs.DirectoryURL, file az var returnValueForError []byte = nil // If we get an error, we just act like there was no content MD5. If validation is set to fail on error, this will fail the transfer of this file later on (at the time of the MD5 check) - // convert format of what we have, if we have something + // convert format of what we have, if we have something in the PathListResponse from Service if file.ContentMD5Base64 != nil { value, err := base64.StdEncoding.DecodeString(*file.ContentMD5Base64) if err != nil { @@ -231,6 +231,10 @@ func getContentMd5(ctx context.Context, directoryURL azbfs.DirectoryURL, file az // Fall back to making a new round trip to the server // This is an interim measure, so that we can still validate MD5s even before they are being returned in the server's // PathList response + // TODO: remove this in a future release, once we know that Service is always returning the MD5s in the PathListResponse. + // Why? Because otherwise, if there's a file with NO MD5, we'll make a round-trip here, but that's pointless if we KNOW that + // that Service is always returning them in the PathListResponse which we've already checked above. + // As at mid-Feb 2019, we don't KNOW that (in fact it's not returning them in the PathListResponse) so we need this code for now. fileURL := directoryURL.FileSystemURL().NewDirectoryURL(*file.Name) props, err := fileURL.GetProperties(ctx) if err != nil { From 6eef4f4c9a9d3e6f88bfc11bc616703e25741e49 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 19 Feb 2019 16:45:39 +1300 Subject: [PATCH 34/64] Fix compilation error in tests --- azbfs/zt_url_file_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azbfs/zt_url_file_test.go b/azbfs/zt_url_file_test.go index 6e0d7c66e..e55126e7f 100644 --- a/azbfs/zt_url_file_test.go +++ b/azbfs/zt_url_file_test.go @@ -193,7 +193,7 @@ func (s *FileURLSuite) TestUploadDownloadRoundTrip(c *chk.C) { c.Assert(pResp.Date(), chk.Not(chk.Equals), "") // Flush data - fResp, err := fileURL.FlushData(context.Background(), 4096) + fResp, err := fileURL.FlushData(context.Background(), 4096, make([]byte, 0)) c.Assert(err, chk.IsNil) c.Assert(fResp.StatusCode(), chk.Equals, http.StatusOK) c.Assert(fResp.ETag(), chk.Not(chk.Equals), "") From ac122236f06f6c17c622f28af1df742c40fee9cb Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 19 Feb 2019 17:37:39 +1300 Subject: [PATCH 35/64] Update contentLength data type in tests --- azbfs/zt_url_file_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azbfs/zt_url_file_test.go b/azbfs/zt_url_file_test.go index e55126e7f..20ac3580c 100644 --- a/azbfs/zt_url_file_test.go +++ b/azbfs/zt_url_file_test.go @@ -206,7 +206,7 @@ func (s *FileURLSuite) TestUploadDownloadRoundTrip(c *chk.C) { resp, err := fileURL.Download(context.Background(), 0, 1024) c.Assert(err, chk.IsNil) c.Assert(resp.StatusCode(), chk.Equals, http.StatusPartialContent) - c.Assert(resp.ContentLength(), chk.Equals, "1024") + c.Assert(resp.ContentLength(), chk.Equals, int64(1024)) c.Assert(resp.ContentType(), chk.Equals, "application/octet-stream") c.Assert(resp.Status(), chk.Not(chk.Equals), "") @@ -219,7 +219,7 @@ func (s *FileURLSuite) TestUploadDownloadRoundTrip(c *chk.C) { resp, err = fileURL.Download(context.Background(), 0, 0) c.Assert(err, chk.IsNil) c.Assert(resp.StatusCode(), chk.Equals, http.StatusOK) - c.Assert(resp.ContentLength(), chk.Equals, "4096") + c.Assert(resp.ContentLength(), chk.Equals, int64(4096)) c.Assert(resp.Date(), chk.Not(chk.Equals), "") c.Assert(resp.ETag(), chk.Not(chk.Equals), "") c.Assert(resp.LastModified(), chk.Not(chk.Equals), "") From a5854debc4f0da1247ef5781fd2aee339dda4a70 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Tue, 19 Feb 2019 19:51:32 +1300 Subject: [PATCH 36/64] Move ListPaths back to FileSystemClient where it belongs The recently-updated Swagger was causing AutoRest to generate ListPaths in the Path client, even though its actually a filesystem operation. In particular, its URL must always point to the file system, using the "directory" parameter to filter to the desired portion of the directory tree. You can't use the actual URL of the directory - so it doesn't belong on Path client (and, for everything except the root, it didn't work there either). --- azbfs/azure_dfs_swagger_manually_edited.json | 16 +-- azbfs/url_directory.go | 5 +- azbfs/zz_generated_filesystem.go | 109 ++++++++++++++++++ azbfs/zz_generated_path.go | 110 ------------------- 4 files changed, 120 insertions(+), 120 deletions(-) diff --git a/azbfs/azure_dfs_swagger_manually_edited.json b/azbfs/azure_dfs_swagger_manually_edited.json index 90b652e4b..62be5c3b2 100644 --- a/azbfs/azure_dfs_swagger_manually_edited.json +++ b/azbfs/azure_dfs_swagger_manually_edited.json @@ -399,7 +399,7 @@ ] }, "get": { - "operationId": "Path_List", + "operationId": "Filesystem_ListPaths", "summary": "List Paths", "description": "List filesystem paths and their properties.", "x-ms-pageable": { @@ -1150,13 +1150,13 @@ "type": "string" }, { - "name": "x-http-method-override", - "description": "Optional. Override the http verb on the service side. Some older http clients do not support PATCH", - "in": "header", - "required": false, - "type": "string" - }, - { + "name": "x-http-method-override", + "description": "Optional. Override the http verb on the service side. Some older http clients do not support PATCH", + "in": "header", + "required": false, + "type": "string" + }, + { "name": "requestBody", "description": "Valid only for append operations. The data to be uploaded and appended to the file.", "in": "body", diff --git a/azbfs/url_directory.go b/azbfs/url_directory.go index df7048acd..997e044e7 100644 --- a/azbfs/url_directory.go +++ b/azbfs/url_directory.go @@ -110,12 +110,13 @@ func (d DirectoryURL) FileSystemURL() FileSystemURL { // Marker) to get the next segment. func (d DirectoryURL) ListDirectorySegment(ctx context.Context, marker *string, recursive bool) (*DirectoryListResponse, error) { // Since listPath is supported on filesystem Url - // covert the directory url to fileSystemUrl + // convert the directory url to fileSystemUrl // and listPath for filesystem with directory path set in the path parameter var maxEntriesInListOperation = int32(1000) - resp, err := d.directoryClient.List(ctx, recursive, d.filesystem, &d.pathParameter, marker, + resp, err := d.FileSystemURL().fileSystemClient.ListPaths(ctx, recursive, d.filesystem, &d.pathParameter, marker, &maxEntriesInListOperation, nil, nil, nil, nil) + return (*DirectoryListResponse)(resp), err } diff --git a/azbfs/zz_generated_filesystem.go b/azbfs/zz_generated_filesystem.go index e331811e3..49aab19ad 100644 --- a/azbfs/zz_generated_filesystem.go +++ b/azbfs/zz_generated_filesystem.go @@ -341,6 +341,115 @@ func (client filesystemClient) listResponder(resp pipeline.Response) (pipeline.R return result, nil } +// ListPaths list filesystem paths and their properties. +// +// recursive is if "true", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If +// "directory" is specified, the list will only include paths that share the same root. filesystem is the filesystem +// identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the +// dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have +// between 3 and 63 characters. directory is filters results to paths within the specified directory. An error occurs +// if the directory does not exist. continuation is the number of paths returned with each invocation is limited. If +// the number of paths to be returned exceeds this limit, a continuation token is returned in the response header +// x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent +// invocation of the list operation to continue listing the paths. maxResults is an optional value that specifies the +// maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items. +// upn is optional. Valid only when Hierarchical Namespace is enabled for the account. If "true", the user identity +// values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory +// Object IDs to User Principal Names. If "false", the values will be returned as Azure Active Directory Object IDs. +// The default value is false. Note that group and application Object IDs are not translated because they do not have +// unique friendly names. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and +// correlation. timeout is an optional operation timeout value in seconds. The period begins when the request is +// received by the service. If the timeout value elapses before the operation completes, the operation fails. xMsDate +// is specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key +// authorization. +func (client filesystemClient) ListPaths(ctx context.Context, recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathList, error) { + if err := validate([]validation{ + {targetValue: maxResults, + constraints: []constraint{{target: "maxResults", name: null, rule: false, + chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, + {targetValue: filesystem, + constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, + {target: "filesystem", name: minLength, rule: 3, chain: nil}, + {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, + {targetValue: xMsClientRequestID, + constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, + chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, + {targetValue: timeout, + constraints: []constraint{{target: "timeout", name: null, rule: false, + chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { + return nil, err + } + req, err := client.listPathsPreparer(recursive, filesystem, directory, continuation, maxResults, upn, xMsClientRequestID, timeout, xMsDate) + if err != nil { + return nil, err + } + resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listPathsResponder}, req) + if err != nil { + return nil, err + } + return resp.(*PathList), err +} + +// listPathsPreparer prepares the ListPaths request. +func (client filesystemClient) listPathsPreparer(recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { + req, err := pipeline.NewRequest("GET", client.url, nil) + if err != nil { + return req, pipeline.NewError(err, "failed to create request") + } + params := req.URL.Query() + if directory != nil && len(*directory) > 0 { + params.Set("directory", *directory) + } + params.Set("recursive", strconv.FormatBool(recursive)) + if continuation != nil && len(*continuation) > 0 { + params.Set("continuation", *continuation) + } + if maxResults != nil { + params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) + } + if upn != nil { + params.Set("upn", strconv.FormatBool(*upn)) + } + params.Set("resource", "filesystem") + if timeout != nil { + params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) + } + req.URL.RawQuery = params.Encode() + if xMsClientRequestID != nil { + req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) + } + if xMsDate != nil { + req.Header.Set("x-ms-date", *xMsDate) + } + req.Header.Set("x-ms-version", ServiceVersion) + return req, nil +} + +// listPathsResponder handles the response to the ListPaths request. +func (client filesystemClient) listPathsResponder(resp pipeline.Response) (pipeline.Response, error) { + err := validateResponse(resp, http.StatusOK) + if resp == nil { + return nil, err + } + result := &PathList{rawResponse: resp.Response()} + if err != nil { + return result, err + } + defer resp.Response().Body.Close() + b, err := ioutil.ReadAll(resp.Response().Body) + if err != nil { + return result, err + } + if len(b) > 0 { + b = removeBOM(b) + err = json.Unmarshal(b, result) + if err != nil { + return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") + } + } + return result, nil +} + // SetProperties set properties for the filesystem. This operation supports conditional HTTP requests. For more // information, see [Specifying Conditional Headers for Blob Service // Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). diff --git a/azbfs/zz_generated_path.go b/azbfs/zz_generated_path.go index 270391e4c..83f376910 100644 --- a/azbfs/zz_generated_path.go +++ b/azbfs/zz_generated_path.go @@ -6,7 +6,6 @@ package azbfs import ( // begin manual edit to generated code "context" - "encoding/json" "github.com/Azure/azure-pipeline-go/pipeline" "io" "io/ioutil" @@ -554,115 +553,6 @@ func (client pathClient) leaseResponder(resp pipeline.Response) (pipeline.Respon return &PathLeaseResponse{rawResponse: resp.Response()}, err } -// List list filesystem paths and their properties. -// -// recursive is if "true", all paths are listed; otherwise, only paths at the root of the filesystem are listed. If -// "directory" is specified, the list will only include paths that share the same root. filesystem is the filesystem -// identifier. The value must start and end with a letter or number and must contain only letters, numbers, and the -// dash (-) character. Consecutive dashes are not permitted. All letters must be lowercase. The value must have -// between 3 and 63 characters. directory is filters results to paths within the specified directory. An error occurs -// if the directory does not exist. continuation is the number of paths returned with each invocation is limited. If -// the number of paths to be returned exceeds this limit, a continuation token is returned in the response header -// x-ms-continuation. When a continuation token is returned in the response, it must be specified in a subsequent -// invocation of the list operation to continue listing the paths. maxResults is an optional value that specifies the -// maximum number of items to return. If omitted or greater than 5,000, the response will include up to 5,000 items. -// upn is optional. Valid only when Hierarchical Namespace is enabled for the account. If "true", the user identity -// values returned in the owner and group fields of each list entry will be transformed from Azure Active Directory -// Object IDs to User Principal Names. If "false", the values will be returned as Azure Active Directory Object IDs. -// The default value is false. Note that group and application Object IDs are not translated because they do not have -// unique friendly names. xMsClientRequestID is a UUID recorded in the analytics logs for troubleshooting and -// correlation. timeout is an optional operation timeout value in seconds. The period begins when the request is -// received by the service. If the timeout value elapses before the operation completes, the operation fails. xMsDate -// is specifies the Coordinated Universal Time (UTC) for the request. This is required when using shared key -// authorization. -func (client pathClient) List(ctx context.Context, recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (*PathList, error) { - if err := validate([]validation{ - {targetValue: maxResults, - constraints: []constraint{{target: "maxResults", name: null, rule: false, - chain: []constraint{{target: "maxResults", name: inclusiveMinimum, rule: 1, chain: nil}}}}}, - {targetValue: filesystem, - constraints: []constraint{{target: "filesystem", name: maxLength, rule: 63, chain: nil}, - {target: "filesystem", name: minLength, rule: 3, chain: nil}, - {target: "filesystem", name: pattern, rule: `^[$a-z0-9][-a-z0-9]{1,61}[a-z0-9]$`, chain: nil}}}, - {targetValue: xMsClientRequestID, - constraints: []constraint{{target: "xMsClientRequestID", name: null, rule: false, - chain: []constraint{{target: "xMsClientRequestID", name: pattern, rule: `^[{(]?[0-9a-f]{8}[-]?([0-9a-f]{4}[-]?){3}[0-9a-f]{12}[)}]?$`, chain: nil}}}}}, - {targetValue: timeout, - constraints: []constraint{{target: "timeout", name: null, rule: false, - chain: []constraint{{target: "timeout", name: inclusiveMinimum, rule: 1, chain: nil}}}}}}); err != nil { - return nil, err - } - req, err := client.listPreparer(recursive, filesystem, directory, continuation, maxResults, upn, xMsClientRequestID, timeout, xMsDate) - if err != nil { - return nil, err - } - resp, err := client.Pipeline().Do(ctx, responderPolicyFactory{responder: client.listResponder}, req) - if err != nil { - return nil, err - } - return resp.(*PathList), err -} - -// listPreparer prepares the List request. -func (client pathClient) listPreparer(recursive bool, filesystem string, directory *string, continuation *string, maxResults *int32, upn *bool, xMsClientRequestID *string, timeout *int32, xMsDate *string) (pipeline.Request, error) { - req, err := pipeline.NewRequest("GET", client.url, nil) - if err != nil { - return req, pipeline.NewError(err, "failed to create request") - } - params := req.URL.Query() - if directory != nil && len(*directory) > 0 { - params.Set("directory", *directory) - } - params.Set("recursive", strconv.FormatBool(recursive)) - if continuation != nil && len(*continuation) > 0 { - params.Set("continuation", *continuation) - } - if maxResults != nil { - params.Set("maxResults", strconv.FormatInt(int64(*maxResults), 10)) - } - if upn != nil { - params.Set("upn", strconv.FormatBool(*upn)) - } - params.Set("resource", "filesystem") - if timeout != nil { - params.Set("timeout", strconv.FormatInt(int64(*timeout), 10)) - } - req.URL.RawQuery = params.Encode() - if xMsClientRequestID != nil { - req.Header.Set("x-ms-client-request-id", *xMsClientRequestID) - } - if xMsDate != nil { - req.Header.Set("x-ms-date", *xMsDate) - } - req.Header.Set("x-ms-version", ServiceVersion) - return req, nil -} - -// listResponder handles the response to the List request. -func (client pathClient) listResponder(resp pipeline.Response) (pipeline.Response, error) { - err := validateResponse(resp, http.StatusOK) - if resp == nil { - return nil, err - } - result := &PathList{rawResponse: resp.Response()} - if err != nil { - return result, err - } - defer resp.Response().Body.Close() - b, err := ioutil.ReadAll(resp.Response().Body) - if err != nil { - return result, err - } - if len(b) > 0 { - b = removeBOM(b) - err = json.Unmarshal(b, result) - if err != nil { - return result, NewResponseError(err, resp.Response(), "failed to unmarshal response body") - } - } - return result, nil -} - // Read read the contents of a file. For read operations, range requests are supported. This operation supports // conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service // Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). From 05df942d591412ef635141d671b81d9ac44e1883 Mon Sep 17 00:00:00 2001 From: John Rusk <25887678+JohnRusk@users.noreply.github.com> Date: Wed, 20 Feb 2019 08:13:12 +1300 Subject: [PATCH 37/64] Fix testSuite content length type for BlobFS --- testSuite/cmd/testblobFS.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/testSuite/cmd/testblobFS.go b/testSuite/cmd/testblobFS.go index b31671013..ad394bacc 100644 --- a/testSuite/cmd/testblobFS.go +++ b/testSuite/cmd/testblobFS.go @@ -8,7 +8,6 @@ import ( "net/url" "os" "path/filepath" - "strconv" "strings" "github.com/Azure/azure-storage-azcopy/azbfs" @@ -96,11 +95,7 @@ func (tbfsc TestBlobFSCommand) verifyRemoteFile() { os.Exit(1) } // get the size of the downloaded file - downloadedLength, err := strconv.ParseInt(dResp.ContentLength(), 10, 64) - if err != nil { - fmt.Println("error converting the content length to int64. failed with error ", err.Error()) - os.Exit(1) - } + downloadedLength := dResp.ContentLength() // open the local file f, err := os.Open(tbfsc.Object) From 6588f817b8f802e337aba6b2c0b49071df9b6e13 Mon Sep 17 00:00:00 2001 From: AK Date: Wed, 20 Feb 2019 14:57:04 -0800 Subject: [PATCH 38/64] Final touch ups for the AzCopyV10 help messages --- cmd/helpMessages.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/helpMessages.go b/cmd/helpMessages.go index 89fba0568..84c253f2d 100644 --- a/cmd/helpMessages.go +++ b/cmd/helpMessages.go @@ -44,7 +44,7 @@ const copyCmdExample = `Upload a single file using OAuth authentication. Please Upload a single file with a SAS: - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Upload a single file with a SAS using a pipeline (block blobs only): +Upload a single file with a SAS using piping (block blobs only): - cat "/path/to/file.txt" | azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" Upload an entire directory with a SAS: @@ -62,7 +62,7 @@ Download a single file using OAuth authentication. Please use 'azcopy login' com Download a single file with a SAS: - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "/path/to/file.txt" -Download a single file with a SAS using a pipeline (block blobs only): +Download a single file with a SAS using piping (block blobs only): - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" > "/path/to/file.txt" Download an entire directory with a SAS: From 79af4a1058d01d3b2b8bba1332ad48693a57b28f Mon Sep 17 00:00:00 2001 From: John Rusk Date: Fri, 22 Feb 2019 13:00:03 +1300 Subject: [PATCH 39/64] Feature/enhanced perf diagnostics (#224) * Show chunk state counts for perf diagnostics * Improve format of perf string * Separate "waiting to be written to disk" from "waiting for prior chunk to arrive" in status tracking * Show chunk status aggregates for both upload and download, with disk bottlneck indicator * Env var to control whether perf states are shown * Include perf states info in log * Update comments and todos * Fix merge error * Streamline shouldShowPerfStates * Add comments * Use named constants is disk constraint tests * Temp disable unreliable test --- cmd/copy.go | 35 ++++- cmd/jobsResume.go | 14 +- cmd/sync.go | 9 +- cmd/zt_sync_download_test.go | 3 +- common/chunkStatusLogger.go | 244 +++++++++++++++++++++++++++++++---- common/chunkedFileWriter.go | 18 ++- common/environment.go | 7 + common/rpc-models.go | 4 + common/singleChunkReader.go | 2 +- ste/init.go | 2 + ste/mgr-JobMgr.go | 81 ++++++++++-- ste/xfer-localToRemote.go | 2 +- ste/xfer-remoteToLocal.go | 2 +- 13 files changed, 371 insertions(+), 52 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index dc21f4c91..627d4fc69 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -783,8 +783,8 @@ func (cca *cookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { } // if json is not desired, and job is done, then we generate a special end message to conclude the job + duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job if jobDone { - duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job exitCode := common.EExitCode.Success() if summary.TransfersFailed > 0 { exitCode = common.EExitCode.Error() @@ -817,24 +817,49 @@ func (cca *cookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { cca.intervalStartTime = time.Now() cca.intervalBytesTransferred = summary.BytesOverWire + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. if throughPut == 0 { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s", + glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s %s", summary.TransfersCompleted, summary.TransfersFailed, summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), summary.TransfersSkipped, summary.TotalTransfers, - scanningString)) + scanningString, + perfString)) } else { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, 2-sec Throughput (Mb/s): %v", + glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s2-sec Throughput (Mb/s): %v%s", summary.TransfersCompleted, summary.TransfersFailed, summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, summary.TotalTransfers, scanningString, ste.ToFixed(throughPut, 4))) + summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, ste.ToFixed(throughPut, 4), diskString)) } } +// Is disk speed looking like a constraint on throughput? Ignore the first little-while, +// to give an (arbitrary) amount of time for things to reach steady-state. +func getPerfDisplayText(perfDiagnosticStrings []string, isDiskConstrained bool, durationOfJob time.Duration) (perfString string, diskString string) { + perfString = "" + if shouldDisplayPerfStates() { + perfString = "[States: " + strings.Join(perfDiagnosticStrings, ", ") + "], " + } + + haveBeenRunningLongEnoughToStabilize := durationOfJob.Seconds() > 30 // this duration is an arbitrary guestimate + if isDiskConstrained && haveBeenRunningLongEnoughToStabilize { + diskString = " (disk may be limiting speed)" + } else { + diskString = "" + } + return +} + +func shouldDisplayPerfStates() bool { + return glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ShowPerfStates()) != "" +} + func isStdinPipeIn() (bool, error) { // check the Stdin to see if we are uploading or downloading info, err := os.Stdin.Stat() diff --git a/cmd/jobsResume.go b/cmd/jobsResume.go index b70b03204..0e2d40d9a 100644 --- a/cmd/jobsResume.go +++ b/cmd/jobsResume.go @@ -85,8 +85,8 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) { jobDone := summary.JobStatus.IsJobDone() // if json is not desired, and job is done, then we generate a special end message to conclude the job + duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job if jobDone { - duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job exitCode := common.EExitCode.Success() if summary.TransfersFailed > 0 { exitCode = common.EExitCode.Error() @@ -118,21 +118,25 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) { cca.intervalStartTime = time.Now() cca.intervalBytesTransferred = summary.BytesOverWire + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. if throughPut == 0 { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s", + glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s %s", summary.TransfersCompleted, summary.TransfersFailed, summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), summary.TransfersSkipped, summary.TotalTransfers, - scanningString)) + scanningString, + perfString)) } else { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped %v Total %s, 2-sec Throughput (Mb/s): %v", + glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped %v Total %s, %s2-sec Throughput (Mb/s): %v%s", summary.TransfersCompleted, summary.TransfersFailed, summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, summary.TotalTransfers, scanningString, ste.ToFixed(throughPut, 4))) + summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, ste.ToFixed(throughPut, 4), diskString)) } } diff --git a/cmd/sync.go b/cmd/sync.go index 63868b2b2..3b18144a7 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -329,8 +329,8 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { } // if json is not desired, and job is done, then we generate a special end message to conclude the job + duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job if jobDone { - duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job exitCode := common.EExitCode.Success() if summary.CopyTransfersFailed+summary.DeleteTransfersFailed > 0 { exitCode = common.EExitCode.Error() @@ -348,11 +348,14 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { summary.JobStatus), exitCode) } - lcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total, 2-sec Throughput (Mb/s): %v", + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + + lcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total%s, 2-sec Throughput (Mb/s): %v%s", summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted, summary.CopyTransfersFailed+summary.DeleteTransfersFailed, summary.CopyTotalTransfers+summary.DeleteTotalTransfers-(summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted+summary.CopyTransfersFailed+summary.DeleteTransfersFailed), - summary.CopyTotalTransfers+summary.DeleteTotalTransfers, ste.ToFixed(throughput, 4))) + summary.CopyTotalTransfers+summary.DeleteTotalTransfers, perfString, ste.ToFixed(throughput, 4), diskString)) } func (cca *cookedSyncCmdArgs) process() (err error) { diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 384601959..a050adf6b 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -180,6 +180,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithEmptyDestination(c *chk.C) { }) } +/* commented out. Ze will put it back in in his next change // regular container->directory sync but destination is identical to the source, transfers are scheduled based on lmt func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) { bsu := getBSU() @@ -222,7 +223,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) }) } - +*/ // regular container->directory sync where destination is missing some files from source, and also has some extra files func (s *cmdIntegrationSuite) TestSyncDownloadWithMismatchedDestination(c *chk.C) { bsu := getBSU() diff --git a/common/chunkStatusLogger.go b/common/chunkStatusLogger.go index 35969697c..3438b3e7f 100644 --- a/common/chunkStatusLogger.go +++ b/common/chunkStatusLogger.go @@ -25,35 +25,114 @@ import ( "fmt" "os" "path" + "sync/atomic" "time" ) +// Identifies a chunk. Always create with NewChunkID type ChunkID struct { Name string OffsetInFile int64 + + // What is this chunk's progress currently waiting on? + // Must be a pointer, because the ChunkID itself is a struct. + // When chunkID is passed around, copies are made, + // but because this is a pointer, all will point to the same + // value for waitReasonIndex (so when we change it, all will see the change) + waitReasonIndex *int32 } -var EWaitReason = WaitReason(0) +func NewChunkID(name string, offsetInFile int64) ChunkID { + dummyWaitReasonIndex := int32(0) + return ChunkID{ + Name: name, + OffsetInFile: offsetInFile, + waitReasonIndex: &dummyWaitReasonIndex, // must initialize, so don't get nil pointer on usage + } +} -type WaitReason string +var EWaitReason = WaitReason{0, ""} + +// WaitReason identifies the one thing that a given chunk is waiting on, at a given moment. +// Basically = state, phrased in terms of "the thing I'm waiting for" +type WaitReason struct { + index int32 + Name string +} -// statuses used by both upload and download -func (WaitReason) RAMToSchedule() WaitReason { return WaitReason("RAM") } // waiting for enough RAM to schedule the chunk -func (WaitReason) WorkerGR() WaitReason { return WaitReason("GR") } // waiting for a goroutine to start running our chunkfunc -func (WaitReason) Body() WaitReason { return WaitReason("Body") } // waiting to finish sending/receiving the BODY -func (WaitReason) Disk() WaitReason { return WaitReason("Disk") } // waiting on disk write to complete -func (WaitReason) ChunkDone() WaitReason { return WaitReason("Done") } // not waiting on anything. Chunk is done. -func (WaitReason) Cancelled() WaitReason { return WaitReason("Cancelled") } // transfer was cancelled. All chunks end with either Done or Cancelled. +// Head (below) has index between GB and Body, just so the ordering is numerical ascending during typical chunk lifetime for both upload and download +func (WaitReason) Nothing() WaitReason { return WaitReason{0, "Nothing"} } // not waiting for anything +func (WaitReason) RAMToSchedule() WaitReason { return WaitReason{1, "RAM"} } // waiting for enough RAM to schedule the chunk +func (WaitReason) WorkerGR() WaitReason { return WaitReason{2, "Worker"} } // waiting for a goroutine to start running our chunkfunc +func (WaitReason) HeaderResponse() WaitReason { return WaitReason{3, "Head"} } // waiting to finish downloading the HEAD +func (WaitReason) Body() WaitReason { return WaitReason{4, "Body"} } // waiting to finish sending/receiving the BODY +func (WaitReason) BodyReReadDueToMem() WaitReason { return WaitReason{5, "BodyReRead-LowRam"} } //waiting to re-read the body after a forced-retry due to low RAM +func (WaitReason) BodyReReadDueToSpeed() WaitReason { return WaitReason{6, "BodyReRead-TooSlow"} } // waiting to re-read the body after a forced-retry due to a slow chunk read (without low RAM) +func (WaitReason) Sorting() WaitReason { return WaitReason{7, "Sorting"} } // waiting for the writer routine, in chunkedFileWriter, to pick up this chunk and sort it into sequence +func (WaitReason) PriorChunk() WaitReason { return WaitReason{8, "Prior"} } // waiting on a prior chunk to arrive (before this one can be saved) +func (WaitReason) QueueToWrite() WaitReason { return WaitReason{9, "Queue"} } // prior chunk has arrived, but is not yet written out to disk +func (WaitReason) DiskIO() WaitReason { return WaitReason{10, "DiskIO"} } // waiting on disk read/write to complete +func (WaitReason) ChunkDone() WaitReason { return WaitReason{11, "Done"} } // not waiting on anything. Chunk is done. +func (WaitReason) Cancelled() WaitReason { return WaitReason{12, "Cancelled"} } // transfer was cancelled. All chunks end with either Done or Cancelled. + +// TODO: consider change the above so that they don't create new struct on every call? Is that necessary/useful? +// Note: reason it's not using the normal enum approach, where it only has a number, is to try to optimize +// the String method below, on the assumption that it will be called a lot. Is that a premature optimization? + +// Upload chunks go through these states, in this order. +// We record this set of states, in this order, so that when we are uploading GetCounts() can return +// counts for only those states that are relevant to upload (some are not relevant, so they are not in this list) +// AND so that GetCounts will return the counts in the order that the states actually happen when uploading. +// That makes it easy for end-users of the counts (i.e. logging and display code) to show the state counts +// in a meaningful left-to-right sequential order. +var uploadWaitReasons = []WaitReason{ + // These first two happen in the transfer initiation function (i.e. the chunkfunc creation loop) + // So their total is constrained to the size of the goroutine pool that runs those functions. + // (e.g. 64, given the GR pool sizing as at Feb 2019) + EWaitReason.RAMToSchedule(), + EWaitReason.DiskIO(), + + // This next one is used when waiting for a worker Go routine to pick up the scheduled chunk func. + // Chunks in this state are effectively a queue of work waiting to be sent over the network + EWaitReason.WorkerGR(), + + // This is the actual network activity + EWaitReason.Body(), // header is not separated out for uploads, so is implicitly included here + // Plus Done/cancelled, which are not included here because not wanted for GetCounts +} -// extra statuses used only by download -func (WaitReason) HeaderResponse() WaitReason { return WaitReason("Head") } // waiting to finish downloading the HEAD -func (WaitReason) BodyReReadDueToMem() WaitReason { return WaitReason("BodyReRead-LowRam") } //waiting to re-read the body after a forced-retry due to low RAM -func (WaitReason) BodyReReadDueToSpeed() WaitReason { return WaitReason("BodyReRead-TooSlow") } // waiting to re-read the body after a forced-retry due to a slow chunk read (without low RAM) -func (WaitReason) WriterChannel() WaitReason { return WaitReason("Writer") } // waiting for the writer routine, in chunkedFileWriter, to pick up this chunk -func (WaitReason) PriorChunk() WaitReason { return WaitReason("Prior") } // waiting on a prior chunk to arrive (before this one can be saved) +// Download chunks go through a larger set of states, due to needing to be re-assembled into sequential order +// See comment on uploadWaitReasons for rationale. +var downloadWaitReasons = []WaitReason{ + // Done by the transfer initiation function (i.e. chunkfunc creation loop) + EWaitReason.RAMToSchedule(), + + // Waiting for a work Goroutine to pick up the chunkfunc and execute it. + // Chunks in this state are effectively a queue of work, waiting for their network downloads to be initiated + EWaitReason.WorkerGR(), + + // These next ones are the actual network activity + EWaitReason.HeaderResponse(), + EWaitReason.Body(), + // next two exist, but are not reported on separately in GetCounts, so are commented out + //EWaitReason.BodyReReadDueToMem(), + //EWaitReason.BodyReReadDueToSpeed(), + + // Sorting and QueueToWrite together comprise a queue of work waiting to be written to disk. + // The former are unsorted, and the latter have been sorted into sequential order. + // PriorChunk is unusual, because chunks in that wait state are not (yet) waiting for their turn to be written to disk, + // instead they are waiting on some prior chunk to finish arriving over the network + EWaitReason.Sorting(), + EWaitReason.PriorChunk(), + EWaitReason.QueueToWrite(), + + // The actual disk write + EWaitReason.DiskIO(), + // Plus Done/cancelled, which are not included here because not wanted for GetCounts +} func (wr WaitReason) String() string { - return string(wr) // avoiding reflection here, for speed, since will be called a lot + return string(wr.Name) // avoiding reflection here, for speed, since will be called a lot } type ChunkStatusLogger interface { @@ -62,34 +141,54 @@ type ChunkStatusLogger interface { type ChunkStatusLoggerCloser interface { ChunkStatusLogger + GetCounts(isDownload bool) []chunkStatusCount + IsDiskConstrained(isUpload, isDownload bool) bool CloseLog() } +// chunkStatusLogger records all chunk state transitions, and makes aggregate data immediately available +// for performance diagnostics. Also optionally logs every individual transition to a file. type chunkStatusLogger struct { - enabled bool + counts []int64 + outputEnabled bool unsavedEntries chan chunkWaitState } -func NewChunkStatusLogger(jobID JobID, logFileFolder string, enable bool) ChunkStatusLoggerCloser { +func NewChunkStatusLogger(jobID JobID, logFileFolder string, enableOutput bool) ChunkStatusLoggerCloser { logger := &chunkStatusLogger{ - enabled: enable, + counts: make([]int64, numWaitReasons()), + outputEnabled: enableOutput, unsavedEntries: make(chan chunkWaitState, 1000000), } - if enable { + if enableOutput { chunkLogPath := path.Join(logFileFolder, jobID.String()+"-chunks.log") // its a CSV, but using log extension for consistency with other files in the directory go logger.main(chunkLogPath) } return logger } +func numWaitReasons() int32 { + return EWaitReason.Cancelled().index + 1 // assume this is the last wait reason +} + +type chunkStatusCount struct { + WaitReason WaitReason + Count int64 +} + type chunkWaitState struct { ChunkID reason WaitReason waitStart time.Time } +//////////////////////////////////// basic functionality ////////////////////////////////// + func (csl *chunkStatusLogger) LogChunkStatus(id ChunkID, reason WaitReason) { - if !csl.enabled { + // always update the in-memory stats, even if output is disabled + csl.countStateTransition(id, reason) + + if !csl.outputEnabled { return } defer func() { @@ -103,7 +202,7 @@ func (csl *chunkStatusLogger) LogChunkStatus(id ChunkID, reason WaitReason) { } func (csl *chunkStatusLogger) CloseLog() { - if !csl.enabled { + if !csl.outputEnabled { return } close(csl.unsavedEntries) @@ -129,6 +228,107 @@ func (csl *chunkStatusLogger) main(chunkLogPath string) { } } +////////////////////////////// aggregate count and analysis support ////////////////////// + +// We maintain running totals of how many chunks are in each state. +// To do so, we must determine the new state (which is simply a parameter) and the old state. +// We obtain and track the old state within the chunkID itself. The alternative, of having a threadsafe +// map in the chunkStatusLogger, to track and look up the states, is considered a risk for performance. +func (csl *chunkStatusLogger) countStateTransition(id ChunkID, newReason WaitReason) { + + // Flip the chunk's state to indicate the new thing that it's waiting for now + oldReasonIndex := atomic.SwapInt32(id.waitReasonIndex, newReason.index) + + // Update the counts + // There's no need to lock the array itself. Instead just do atomic operations on the contents. + // (See https://groups.google.com/forum/#!topic/Golang-nuts/Ud4Dqin2Shc) + if oldReasonIndex > 0 && oldReasonIndex < int32(len(csl.counts)) { + atomic.AddInt64(&csl.counts[oldReasonIndex], -1) + } + if newReason.index < int32(len(csl.counts)) { + atomic.AddInt64(&csl.counts[newReason.index], 1) + } +} + +func (csl *chunkStatusLogger) getCount(reason WaitReason) int64 { + return atomic.LoadInt64(&csl.counts[reason.index]) +} + +// Gets the current counts of chunks in each wait state +// Intended for performance diagnostics and reporting +func (csl *chunkStatusLogger) GetCounts(isDownload bool) []chunkStatusCount { + + var allReasons []WaitReason + if isDownload { + allReasons = downloadWaitReasons + } else { + allReasons = uploadWaitReasons + } + + result := make([]chunkStatusCount, len(allReasons)) + for i, reason := range allReasons { + count := csl.getCount(reason) + + // for simplicity in consuming the results, all the body read states are rolled into one here + if reason == EWaitReason.BodyReReadDueToSpeed() || reason == EWaitReason.BodyReReadDueToMem() { + panic("body re-reads should not be requested in counts. They get rolled into the main Body one") + } + if reason == EWaitReason.Body() { + count += csl.getCount(EWaitReason.BodyReReadDueToSpeed()) + count += csl.getCount(EWaitReason.BodyReReadDueToMem()) + } + + result[i] = chunkStatusCount{reason, count} + } + return result +} + +func (csl *chunkStatusLogger) IsDiskConstrained(isUpload, isDownload bool) bool { + if isUpload { + return csl.isUploadDiskConstrained() + } else if isDownload { + return csl.isDownloadDiskConstrained() + } else { + return false // it's neither upload nor download (e.g. S2S) + } +} + +// is disk the bottleneck in an upload? +func (csl *chunkStatusLogger) isUploadDiskConstrained() bool { + // If we are uploading, and there's almost nothing waiting to go out over the network, then + // probably the reason there's not much queued is that the disk is slow. + // BTW, we can't usefully look at any of the _earlier_ states, because they happen in the _generation_ of the chunk funcs + // (not the _execution_ and so their counts will just tend to equal that of the small goroutine pool that runs them). + // It might be convenient if we could compare TWO queue sizes here, as we do in isDownloadDiskConstrained, but unfortunately our + // Jan 2019 architecture only gives us ONE useful queue-like state when uploading, so we can't compare two. + const nearZeroQueueSize = 10 // TODO: is there any intelligent way to set this threshold? It's just an arbitrary guestimate of "small" at the moment + queueForNetworkIsSmall := csl.getCount(EWaitReason.WorkerGR()) < nearZeroQueueSize + + beforeGRWaitQueue := csl.getCount(EWaitReason.RAMToSchedule()) + csl.getCount(EWaitReason.DiskIO()) + areStillReadingDisk := beforeGRWaitQueue > 0 // size of queue for network is irrelevant if we are no longer actually reading disk files, and therefore no longer putting anything into the queue for network + + return areStillReadingDisk && queueForNetworkIsSmall +} + +// is disk the bottleneck in a download? +func (csl *chunkStatusLogger) isDownloadDiskConstrained() bool { + // See how many chunks are waiting on the disk. I.e. are queued before the actual disk state. + // Don't include the "PriorChunk" state, because that's not actually waiting on disk at all, it + // can mean waiting on network and/or waiting-on-Storage-Service. We don't know which. So we just exclude it from consideration. + chunksWaitingOnDisk := csl.getCount(EWaitReason.Sorting()) + csl.getCount(EWaitReason.QueueToWrite()) + + // i.e. are queued before the actual network states + chunksWaitingOnNetwork := csl.getCount(EWaitReason.WorkerGR()) + + // if we have way more stuff waiting on disk than on network, we can assume disk is the bottleneck + const activeDiskQThreshold = 10 + const bigDifference = 5 // TODO: review/tune the arbitrary constant here + return chunksWaitingOnDisk > activeDiskQThreshold && // this test is in case both are near zero, as they would be near the end of the job + chunksWaitingOnDisk > bigDifference*chunksWaitingOnNetwork +} + +///////////////////////////////////// Sample LinqPad query for manual analysis of chunklog ///////////////////////////////////// + /* LinqPad query used to analyze/visualize the CSV as is follows: Needs CSV driver for LinqPad to open the CSV - e.g. https://github.com/dobrou/CsvLINQPadDriver diff --git a/common/chunkedFileWriter.go b/common/chunkedFileWriter.go index d048c194d..e84405426 100644 --- a/common/chunkedFileWriter.go +++ b/common/chunkedFileWriter.go @@ -161,7 +161,7 @@ func (w *chunkedFileWriter) EnqueueChunk(ctx context.Context, id ChunkID, chunkS } // enqueue it - w.chunkLogger.LogChunkStatus(id, EWaitReason.WriterChannel()) + w.chunkLogger.LogChunkStatus(id, EWaitReason.Sorting()) select { case err = <-w.failureError: if err != nil { @@ -237,6 +237,7 @@ func (w *chunkedFileWriter) workerRoutine(ctx context.Context) { w.chunkLogger.LogChunkStatus(newChunk.id, EWaitReason.PriorChunk()) // may have to wait on prior chunks to arrive // Process all chunks that we can + w.setStatusForContiguousAvailableChunks(unsavedChunksByFileOffset, nextOffsetToSave) // update states of those that have all their prior ones already here err := w.sequentiallyProcessAvailableChunks(unsavedChunksByFileOffset, &nextOffsetToSave, md5Hasher) if err != nil { w.failureError <- err @@ -268,6 +269,19 @@ func (w *chunkedFileWriter) sequentiallyProcessAvailableChunks(unsavedChunksByFi } } +// Advances the status of chunks which are no longer waiting on missing predecessors, but are instead just waiting on +// us to get around to (sequentially) saving them +func (w *chunkedFileWriter) setStatusForContiguousAvailableChunks(unsavedChunksByFileOffset map[int64]fileChunk, nextOffsetToSave int64) { + for { + nextChunkInSequence, exists := unsavedChunksByFileOffset[nextOffsetToSave] + if !exists { + return //its not there yet, so no need to touch anything AFTER it. THEY are still waiting for prior chunk + } + nextOffsetToSave += int64(len(nextChunkInSequence.data)) + w.chunkLogger.LogChunkStatus(nextChunkInSequence.id, EWaitReason.QueueToWrite()) // we WILL write this. Just may have to write others before it + } +} + // Saves one chunk to its destination func (w *chunkedFileWriter) saveOneChunk(chunk fileChunk) error { defer func() { @@ -277,7 +291,7 @@ func (w *chunkedFileWriter) saveOneChunk(chunk fileChunk) error { w.chunkLogger.LogChunkStatus(chunk.id, EWaitReason.ChunkDone()) // this chunk is all finished }() - w.chunkLogger.LogChunkStatus(chunk.id, EWaitReason.Disk()) + w.chunkLogger.LogChunkStatus(chunk.id, EWaitReason.DiskIO()) _, err := w.file.Write(chunk.data) // unlike Read, Write must process ALL the data, or have an error. It can't return "early". if err != nil { return err diff --git a/common/environment.go b/common/environment.go index e9b9eb37e..e8fd0e558 100644 --- a/common/environment.go +++ b/common/environment.go @@ -62,3 +62,10 @@ func (EnvironmentVariable) ProfileCPU() EnvironmentVariable { func (EnvironmentVariable) ProfileMemory() EnvironmentVariable { return EnvironmentVariable{Name: "AZCOPY_PROFILE_MEM"} } + +func (EnvironmentVariable) ShowPerfStates() EnvironmentVariable { + return EnvironmentVariable{ + Name: "AZCOPY_SHOW_PERF_STATES", + Description: "If set, to anything, on-screen output will include counts of chunks by state", + } +} diff --git a/common/rpc-models.go b/common/rpc-models.go index 0cb94bbd2..34bf4ad50 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -133,6 +133,8 @@ type ListJobSummaryResponse struct { TotalBytesEnumerated uint64 FailedTransfers []TransferDetail SkippedTransfers []TransferDetail + IsDiskConstrained bool + PerfStrings []string } // represents the JobProgressPercentage Summary response for list command when requested the Job Progress Summary for given JobId @@ -153,6 +155,8 @@ type ListSyncJobSummaryResponse struct { DeleteTransfersCompleted uint32 DeleteTransfersFailed uint32 FailedTransfers []TransferDetail + IsDiskConstrained bool + PerfStrings []string } type ListJobTransfersRequest struct { diff --git a/common/singleChunkReader.go b/common/singleChunkReader.go index 39761a677..3ef939f12 100644 --- a/common/singleChunkReader.go +++ b/common/singleChunkReader.go @@ -200,7 +200,7 @@ func (cr *singleChunkReader) blockingPrefetch(fileReader io.ReaderAt, isRetry bo cr.buffer = cr.slicePool.RentSlice(uint32Checked(cr.length)) // read bytes into the buffer - cr.chunkLogger.LogChunkStatus(cr.chunkId, EWaitReason.Disk()) + cr.chunkLogger.LogChunkStatus(cr.chunkId, EWaitReason.DiskIO()) totalBytesRead, err := fileReader.ReadAt(cr.buffer, cr.chunkId.OffsetInFile) if err != nil && err != io.EOF { return err diff --git a/ste/init.go b/ste/init.go index 1aececbb3..f05b3b729 100644 --- a/ste/init.go +++ b/ste/init.go @@ -458,6 +458,7 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse { // Get the number of active go routines performing the transfer or executing the chunk Func // TODO: added for debugging purpose. remove later js.ActiveConnections = jm.ActiveConnections() + js.PerfStrings, js.IsDiskConstrained = jm.GetPerfInfo() // If the status is cancelled, then no need to check for completerJobOrdered // since user must have provided the consent to cancel an incompleteJob if that @@ -583,6 +584,7 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { // Get the number of active go routines performing the transfer or executing the chunk Func // TODO: added for debugging purpose. remove later js.ActiveConnections = jm.ActiveConnections() + js.PerfStrings, js.IsDiskConstrained = jm.GetPerfInfo() // If the status is cancelled, then no need to check for completerJobOrdered // since user must have provided the consent to cancel an incompleteJob if that diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index f099090a3..f6a799fba 100644 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -23,6 +23,7 @@ package ste import ( "context" "fmt" + "strings" "sync" "sync/atomic" @@ -65,6 +66,7 @@ type IJobMgr interface { ReleaseAConnection() // TODO: added for debugging purpose. remove later ActiveConnections() int64 + GetPerfInfo() (displayStrings []string, isDiskConstrained bool) //Close() getInMemoryTransitJobState() InMemoryTransitJobState // get in memory transit job state saved in this job. setInMemoryTransitJobState(state InMemoryTransitJobState) // set in memory transit job state saved in this job. @@ -77,10 +79,10 @@ type IJobMgr interface { func newJobMgr(appLogger common.ILogger, jobID common.JobID, appCtx context.Context, level common.LogLevel, commandString string, logFileFolder string) IJobMgr { // atomicAllTransfersScheduled is set to 1 since this api is also called when new job part is ordered. - enableChunkLog := level.ToPipelineLogLevel() == pipeline.LogDebug + enableChunkLogOutput := level.ToPipelineLogLevel() == pipeline.LogDebug jm := jobMgr{jobID: jobID, jobPartMgrs: newJobPartToJobPartMgr(), include: map[string]int{}, exclude: map[string]int{}, - logger: common.NewJobLogger(jobID, level, appLogger, logFileFolder), - chunkStatusLogger: common.NewChunkStatusLogger(jobID, logFileFolder, enableChunkLog), + logger: common.NewJobLogger(jobID, level, appLogger, logFileFolder), + chunkStatusLogger: common.NewChunkStatusLogger(jobID, logFileFolder, enableChunkLogOutput), /*Other fields remain zero-value until this job is scheduled */} jm.reset(appCtx, commandString) return &jm @@ -103,11 +105,11 @@ func (jm *jobMgr) reset(appCtx context.Context, commandString string) IJobMgr { // jobMgr represents the runtime information for a Job type jobMgr struct { - logger common.ILoggerResetable + logger common.ILoggerResetable chunkStatusLogger common.ChunkStatusLoggerCloser - jobID common.JobID // The Job's unique ID - ctx context.Context - cancel context.CancelFunc + jobID common.JobID // The Job's unique ID + ctx context.Context + cancel context.CancelFunc jobPartMgrs jobPartToJobPartMgr // The map of part #s to JobPartMgrs // partsDone keep the count of completed part of the Job. @@ -128,6 +130,8 @@ type jobMgr struct { // atomicCurrentConcurrentConnections defines the number of active goroutines performing the transfer / executing the chunk func // TODO: added for debugging purpose. remove later atomicCurrentConcurrentConnections int64 + atomicIsUploadIndicator int32 + atomicIsDownloadIndicator int32 } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -165,16 +169,57 @@ func (jm *jobMgr) ActiveConnections() int64 { return atomic.LoadInt64(&jm.atomicCurrentConcurrentConnections) } +// GetPerfStrings returns strings that may be logged for performance diagnostic purposes +// The number and content of strings may change as we enhance our perf diagnostics +func (jm *jobMgr) GetPerfInfo() (displayStrings []string, isDiskConstrained bool) { + + // get data appropriate to our current transfer direction + isUpload := atomic.LoadInt32(&jm.atomicIsUploadIndicator) == 1 + isDownload := atomic.LoadInt32(&jm.atomicIsDownloadIndicator) == 1 + chunkStateCounts := jm.chunkStatusLogger.GetCounts(isDownload) + + // convert the counts to simple strings for consumption by callers + const format = "%c: %2d" + result := make([]string, len(chunkStateCounts)+1) + total := int64(0) + for i, c := range chunkStateCounts { + result[i] = fmt.Sprintf(format, c.WaitReason.Name[0], c.Count) + total += c.Count + } + result[len(result)-1] = fmt.Sprintf(format, 'T', total) + + diskCon := jm.chunkStatusLogger.IsDiskConstrained(isUpload, isDownload) + + // logging from here is a bit of a hack + // TODO: can we find a better way to get this info into the log? The caller is at app level, + // not job level, so can't log it directly AFAICT. + jm.logPerfInfo(result, diskCon) + + return result, diskCon +} + +func (jm *jobMgr) logPerfInfo(displayStrings []string, isDiskConstrained bool) { + var diskString string + if isDiskConstrained { + diskString = "disk MAY BE limiting throughput" + } else { + diskString = "disk IS NOT limiting throughput" + } + msg := fmt.Sprintf("PERF: disk %s. States: %s", diskString, strings.Join(displayStrings, ", ")) + jm.Log(pipeline.LogInfo, msg) +} + // initializeJobPartPlanInfo func initializes the JobPartPlanInfo handler for given JobPartOrder func (jm *jobMgr) AddJobPart(partNum PartNumber, planFile JobPartPlanFileName, sourceSAS string, destinationSAS string, scheduleTransfers bool) IJobPartMgr { jpm := &jobPartMgr{jobMgr: jm, filename: planFile, sourceSAS: sourceSAS, destinationSAS: destinationSAS, pacer: JobsAdmin.(*jobsAdmin).pacer, - slicePool: JobsAdmin.(*jobsAdmin).slicePool, + slicePool: JobsAdmin.(*jobsAdmin).slicePool, cacheLimiter: JobsAdmin.(*jobsAdmin).cacheLimiter} jpm.planMMF = jpm.filename.Map() jm.jobPartMgrs.Set(partNum, jpm) jm.finalPartOrdered = jpm.planMMF.Plan().IsFinalPart + jm.setDirection(jpm.Plan().FromTo) if scheduleTransfers { // If the schedule transfer is set to true // Instead of the scheduling the Transfer for given JobPart @@ -186,6 +231,21 @@ func (jm *jobMgr) AddJobPart(partNum PartNumber, planFile JobPartPlanFileName, s return jpm } +// Remembers which direction we are running in (upload, download or neither (for service to service)) +// It actually remembers the direction that our most recently-added job PART is running in, +// because that's where the fromTo information can be found, +// but we assume taht all the job parts are running in the same direction +func (jm *jobMgr) setDirection(fromTo common.FromTo) { + fromIsLocal := fromTo.From() == common.ELocation.Local() + toIsLocal := fromTo.To() == common.ELocation.Local() + + isUpload := fromIsLocal && !toIsLocal + isDownload := !fromIsLocal && toIsLocal + + atomic.StoreInt32(&jm.atomicIsUploadIndicator, common.Iffint32(isUpload, 1, 0)) + atomic.StoreInt32(&jm.atomicIsDownloadIndicator, common.Iffint32(isDownload, 1, 0)) +} + // SetIncludeExclude sets the include / exclude list of transfers // supplied with resume command to include or exclude mentioned transfers func (jm *jobMgr) SetIncludeExclude(include, exclude map[string]int) { @@ -280,16 +340,15 @@ func (jm *jobMgr) PipelineLogInfo() pipeline.LogOptions { } } func (jm *jobMgr) Panic(err error) { jm.logger.Panic(err) } -func (jm *jobMgr) CloseLog(){ +func (jm *jobMgr) CloseLog() { jm.logger.CloseLog() jm.chunkStatusLogger.CloseLog() } -func (jm *jobMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason){ +func (jm *jobMgr) LogChunkStatus(id common.ChunkID, reason common.WaitReason) { jm.chunkStatusLogger.LogChunkStatus(id, reason) } - // PartsDone returns the number of the Job's parts that are either completed or failed //func (jm *jobMgr) PartsDone() uint32 { return atomic.LoadUint32(&jm.partsDone) } diff --git a/ste/xfer-localToRemote.go b/ste/xfer-localToRemote.go index ce1e1a9de..d02cb61dc 100644 --- a/ste/xfer-localToRemote.go +++ b/ste/xfer-localToRemote.go @@ -139,7 +139,7 @@ func scheduleUploadChunks(jptm IJobPartTransferMgr, srcName string, srcFile comm safeToUseHash := true for startIndex := int64(0); startIndex < fileSize || isDummyChunkInEmptyFile(startIndex, fileSize); startIndex += int64(chunkSize) { - id := common.ChunkID{Name: srcName, OffsetInFile: startIndex} + id := common.NewChunkID(srcName, startIndex) adjustedChunkSize := int64(chunkSize) // compute actual size of the chunk diff --git a/ste/xfer-remoteToLocal.go b/ste/xfer-remoteToLocal.go index a3ffcefdd..2609a9657 100644 --- a/ste/xfer-remoteToLocal.go +++ b/ste/xfer-remoteToLocal.go @@ -136,7 +136,7 @@ func remoteToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, chunkCount := uint32(0) for startIndex := int64(0); startIndex < fileSize; startIndex += downloadChunkSize { - id := common.ChunkID{Name: info.Destination, OffsetInFile: startIndex} + id := common.NewChunkID(info.Destination, startIndex) adjustedChunkSize := downloadChunkSize // compute exact size of the chunk From 1819b411d7c3c2ac990fa27670acc55a07a7d2a0 Mon Sep 17 00:00:00 2001 From: John Rusk Date: Fri, 22 Feb 2019 14:07:22 +1300 Subject: [PATCH 40/64] Fix/prevent double counting (#227) * Prevent double-counting of transfer completion * Protect against double-counting of chunk completions --- common/chunkStatusLogger.go | 25 ++++++++++++++++++++++--- ste/mgr-JobPartTransferMgr.go | 33 ++++++++++++++++++++++++++++++--- ste/uploader.go | 2 +- ste/xfer-URLToBlob.go | 6 +++--- 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/common/chunkStatusLogger.go b/common/chunkStatusLogger.go index 3438b3e7f..8cb6a6bf3 100644 --- a/common/chunkStatusLogger.go +++ b/common/chunkStatusLogger.go @@ -40,14 +40,33 @@ type ChunkID struct { // but because this is a pointer, all will point to the same // value for waitReasonIndex (so when we change it, all will see the change) waitReasonIndex *int32 + + // Like waitReasonIndex, but is effectively just a boolean to track whether we are done. + // Must be a pointer, for same reason that waitReasonIndex is. + // Can't be done just off waitReasonIndex because for downloads we actually + // tell the jptm we are done before the chunk has been flushed out to disk, so + // waitReasonIndex isn't yet ready to go to "Done" at that time. + completionNotifiedToJptm *int32 + + // TODO: it's a bit odd having two pointers in a struct like this. Review, maybe we should always work + // with pointers to chunk ids, with nocopy? If we do that, the two fields that are currently pointers + // can become non-pointers } func NewChunkID(name string, offsetInFile int64) ChunkID { dummyWaitReasonIndex := int32(0) + zeroNotificationState := int32(0) return ChunkID{ - Name: name, - OffsetInFile: offsetInFile, - waitReasonIndex: &dummyWaitReasonIndex, // must initialize, so don't get nil pointer on usage + Name: name, + OffsetInFile: offsetInFile, + waitReasonIndex: &dummyWaitReasonIndex, // must initialize, so don't get nil pointer on usage + completionNotifiedToJptm: &zeroNotificationState, + } +} + +func (id ChunkID) SetCompletionNotificationSent() { + if atomic.SwapInt32(id.completionNotifiedToJptm, 1) != 0 { + panic("cannot complete the same chunk twice") } } diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index 90057fac5..98ae9771b 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -30,7 +30,8 @@ type IJobPartTransferMgr interface { CacheLimiter() common.CacheLimiter StartJobXfer() IsForceWriteTrue() bool - ReportChunkDone() (lastChunk bool, chunksDone uint32) + ReportChunkDone(id common.ChunkID) (lastChunk bool, chunksDone uint32) + UnsafeReportChunkDone() (lastChunk bool, chunksDone uint32) TransferStatus() common.TransferStatus SetStatus(status common.TransferStatus) SetErrorCode(errorCode int32) @@ -101,6 +102,9 @@ type jobPartTransferMgr struct { // NumberOfChunksDone determines the final cancellation or completion of a transfer atomicChunksDone uint32 + // used defensively to protect against accidental double counting + atomicCompletionIndicator uint32 + /* @Parteek removed 3/23 morning, as jeff ad equivalent // transfer chunks are put into this channel and execution engine takes chunk out of this channel. @@ -240,7 +244,15 @@ func (jptm *jobPartTransferMgr) SetActionAfterLastChunk(f func()) { } // Call Done when a chunk has completed its transfer; this method returns the number of chunks completed so far -func (jptm *jobPartTransferMgr) ReportChunkDone() (lastChunk bool, chunksDone uint32) { +func (jptm *jobPartTransferMgr) ReportChunkDone(id common.ChunkID) (lastChunk bool, chunksDone uint32) { + + // Tell the id to remember that we (the jptm) have been told about its completion + // Will panic if we've already been told about its completion before. + // Why? As defensive programming, since if we accidentally counted one chunk twice, we'd complete + // before another was finish. Which would be bad + id.SetCompletionNotificationSent() + + // Do our actual processing chunksDone = atomic.AddUint32(&jptm.atomicChunksDone, 1) lastChunk = chunksDone == jptm.numChunks if lastChunk { @@ -249,6 +261,11 @@ func (jptm *jobPartTransferMgr) ReportChunkDone() (lastChunk bool, chunksDone ui return lastChunk, chunksDone } +// TODO: phase this method out. It's just here to support parts of the codebase that don't yet have chunk IDs +func (jptm *jobPartTransferMgr) UnsafeReportChunkDone() (lastChunk bool, chunksDone uint32) { + return jptm.ReportChunkDone(common.NewChunkID("", 0)) +} + // If an automatic action has been specified for after the last chunk, run it now // (Prior to introduction of this routine, individual chunkfuncs had to check the return values // of ReportChunkDone and then implement their own versions of the necessary transfer epilogue code. @@ -437,10 +454,20 @@ func (jptm *jobPartTransferMgr) Panic(err error) { jptm.jobPartMgr.Panic(err) } // Call ReportTransferDone to report when a Transfer for this Job Part has completed // TODO: I feel like this should take the status & we kill SetStatus -// TODO: also, it looks like if we accidentally call this twice, on the one jptm, it just treats that as TWO successful transfers, which is a bug func (jptm *jobPartTransferMgr) ReportTransferDone() uint32 { // In case of context leak in job part transfer manager. jptm.Cancel() + // defensive programming check, to make sure this method is not called twice for the same transfer + // (since if it was, job would count us as TWO completions, and maybe miss another transfer that + // should have been counted but wasn't) + // TODO: it would be nice if this protection was actually in jobPartMgr.ReportTransferDone, + // but that's harder to implement (would imply need for a threadsafe map there, to track + // status by transfer). So for now we are going with the check here. This is the only call + // to the jobPartManager anyway (as it Feb 2019) + if atomic.SwapUint32(&jptm.atomicCompletionIndicator, 1) != 0 { + panic("cannot report the same transfer done twice") + } + return jptm.jobPartMgr.ReportTransferDone() } diff --git a/ste/uploader.go b/ste/uploader.go index 57a00dd83..7de937d2a 100644 --- a/ste/uploader.go +++ b/ste/uploader.go @@ -112,7 +112,7 @@ func createChunkFunc(setDoneStatusOnExit bool, jptm IJobPartTransferMgr, id comm return func(workerId int) { // BEGIN standard prefix that all chunk funcs need - defer jptm.ReportChunkDone() // whether successful or failed, it's always "done" and we must always tell the jptm + defer jptm.ReportChunkDone(id) // whether successful or failed, it's always "done" and we must always tell the jptm jptm.OccupyAConnection() // TODO: added the two operations for debugging purpose. remove later defer jptm.ReleaseAConnection() diff --git a/ste/xfer-URLToBlob.go b/ste/xfer-URLToBlob.go index ee3d19c4e..af8e51ede 100644 --- a/ste/xfer-URLToBlob.go +++ b/ste/xfer-URLToBlob.go @@ -229,7 +229,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd if bbc.jptm.ShouldLog(pipeline.LogDebug) { bbc.jptm.Log(pipeline.LogDebug, fmt.Sprintf("Transfer cancelled. not picking up chunk %d", chunkId)) } - if lastChunk, _ := bbc.jptm.ReportChunkDone(); lastChunk { + if lastChunk, _ := bbc.jptm.UnsafeReportChunkDone(); lastChunk { if bbc.jptm.ShouldLog(pipeline.LogDebug) { bbc.jptm.Log(pipeline.LogDebug, "Finalizing transfer cancellation") } @@ -266,7 +266,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd } } - if lastChunk, _ := bbc.jptm.ReportChunkDone(); lastChunk { + if lastChunk, _ := bbc.jptm.UnsafeReportChunkDone(); lastChunk { if bbc.jptm.ShouldLog(pipeline.LogDebug) { bbc.jptm.Log(pipeline.LogDebug, "Finalizing transfer cancellation") } @@ -276,7 +276,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd } // step 4: check if this is the last chunk - if lastChunk, _ := bbc.jptm.ReportChunkDone(); lastChunk { + if lastChunk, _ := bbc.jptm.UnsafeReportChunkDone(); lastChunk { // If the transfer gets cancelled before the putblock list if bbc.jptm.WasCanceled() { transferDone() From 9a6afb316956bf5866bbb81f3e6a383182ddbbcf Mon Sep 17 00:00:00 2001 From: John Rusk Date: Fri, 22 Feb 2019 17:18:11 +1300 Subject: [PATCH 41/64] Reduce memory usage, especially with many small files (#228) * Constrain RAM usage by the multiSizeSlicePool Specifically: Reduce max slice count in medium and large sub pools. No need to reduce it in the really tiny pools, because even at their (unchanged) original max size, they can't consume much RAM. This reduction of max count in the medium-sized pools is expected to help with the out-of-RAM with small files bug. And change GC threshold to something more approprinate for our app. This is also expected to help with the out-of-RAM with small files bug And gradulally drain unused large pools. This is expected to help with a related but different anticipated case, where there are large files with differing chunk sizes in the job. * Tweak memory constants Move GCPercent back up a bit, to be less agressive with it, and activate the new setting later. Tweak per CPU allocation down slightly * Fix typos in comments --- common/multiSizeSlicePool.go | 47 +++++++++++++++++++++++++++++++++++- main.go | 14 +++++++++++ ste/JobsAdmin.go | 40 +++++++++++++++++++++++------- 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/common/multiSizeSlicePool.go b/common/multiSizeSlicePool.go index cfb687d1e..f0e0a40c3 100644 --- a/common/multiSizeSlicePool.go +++ b/common/multiSizeSlicePool.go @@ -29,6 +29,7 @@ import ( type ByteSlicePooler interface { RentSlice(desiredLength uint32) []byte ReturnSlice(slice []byte) + Prune() } // Pools byte slices of a single size. @@ -86,11 +87,16 @@ func NewMultiSizeSlicePool(maxSliceLength uint32) ByteSlicePooler { maxSlotIndex, _ := getSlotInfo(maxSliceLength) poolsBySize := make([]*simpleSlicePool, maxSlotIndex+1) for i := 0; i <= maxSlotIndex; i++ { - poolsBySize[i] = newSimpleSlicePool(1000) // TODO: review capacity (setting too low doesn't break anything, since we don't block when full, so maybe only 100 or so is OK?) + maxCount := getMaxSliceCountInPool(i) + poolsBySize[i] = newSimpleSlicePool(maxCount) } return &multiSizeSlicePool{poolsBySize: poolsBySize} } +var indexOf32KSlot, _ = getSlotInfo(32 * 1024) + +// For a given requested len(slice), this returns the slot index to use, and the max +// cap(slice) of the slices that will be found at that index func getSlotInfo(exactSliceLength uint32) (slotIndex int, maxCapInSlot int) { if exactSliceLength <= 0 { panic("exact slice length must be greater than zero") @@ -121,6 +127,24 @@ func getSlotInfo(exactSliceLength uint32) (slotIndex int, maxCapInSlot int) { return } +func holdsSmallSlices(slotIndex int) bool { + return slotIndex <= indexOf32KSlot +} + +// For a given slot index, this returns the max number of pooled slices which the pool at +// that index should be allowed to hold. +func getMaxSliceCountInPool(slotIndex int) int { + if holdsSmallSlices(slotIndex) { + // Choose something fairly high for these because there's no significant RAM + // cost in doing so, and small files are a tricky case for perf so let's give + // them all the pooling help we can + return 500 + } else { + // Limit the medium and large ones a bit more strictly. + return 100 + } +} + // RentSlice borrows a slice from the pool (or creates a new one if none of suitable capacity is available) // Note that the returned slice may contain non-zero data - i.e. old data from the previous time it was used. // That's safe IFF you are going to do the likes of io.ReadFull to read into it, since you know that all of the @@ -161,3 +185,24 @@ func (mp *multiSizeSlicePool) ReturnSlice(slice []byte) { // put the slice back into the pool pool.Put(slice) } + +// Prune inactive stuff in all the big slots if due (don't worry about the little ones, they don't eat much RAM) +// Why do this? Because for the large slot sizes its hard to deal with slots that are full and IDLE. +// I.e. we were using them, but now we're working with other files in the same job that have different chunk sizes, +// so we have a pool slot that's full of slices that are no longer getting used. +// Would it work to just give the slot a very small max capacity? Maybe. E.g. max number in slot = 4 for the 128 MB slot. +// But can we be confident that 4 is enough for the pooling to have the desired benefits? Not sure. +// Hence the pruning, so that we don't need to set the fixed limits that low. With pruning, the count will (gradually) +// come down only when the slot is IDLE. +func (mp *multiSizeSlicePool) Prune() { + for index := 0; index < len(mp.poolsBySize); index++ { + shouldPrune := !holdsSmallSlices(index) + if shouldPrune { + // Get one item from the pool and throw it away. + // With repeated calls of Prune, this will gradually drain idle pools. + // But, since Prune is not called very often, + // it won't have much adverse impact on active pools. + _ = mp.poolsBySize[index].Get() + } + } +} diff --git a/main.go b/main.go index 0a162b385..4cbc345af 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,8 @@ import ( "log" "os" "runtime" + "runtime/debug" + "time" ) // get the lifecycle manager to print messages @@ -51,7 +53,19 @@ func main() { log.Fatalf("initialization failed: %v", err) } + configureGC() + ste.MainSTE(common.ComputeConcurrencyValue(runtime.NumCPU()), 2400, azcopyAppPathFolder, azcopyLogPathFolder) cmd.Execute(azcopyAppPathFolder, azcopyLogPathFolder) glcm.Exit("", common.EExitCode.Success()) } + +// Golang's default behaviour is to GC when new objects = (100% of) total of objects surviving previous GC. +// But our "survivors" add up to many GB, so its hard for users to be confident that we don't have +// a memory leak (since with that default setting new GCs are very rare in our case). So configure them to be more frequent. +func configureGC() { + go func() { + time.Sleep(20 * time.Second) // wait a little, so that our initial pool of buffers can get allocated without heaps of (unnecessary) GC activity + debug.SetGCPercent(20) // activate more aggressive/frequent GC than the default + }() +} diff --git a/ste/JobsAdmin.go b/ste/JobsAdmin.go index 9ddbdb2f4..06c8ac10a 100644 --- a/ste/JobsAdmin.go +++ b/ste/JobsAdmin.go @@ -130,10 +130,12 @@ func initJobsAdmin(appCtx context.Context, concurrentConnections int, targetRate // TODO: make ram usage configurable, with the following as just the default // Decide on a max amount of RAM we are willing to use. This functions as a cap, and prevents excessive usage. // There's no measure of physical RAM in the STD library, so we guestimate conservatively, based on CPU count (logical, not phyiscal CPUs) - const gbToUsePerCpu = 0.6 // should be enough to support the amount of traffic 1 CPU can drive, and also less than the typical installed RAM-per-CPU + // Note that, as at Feb 2019, the multiSizeSlicePooler uses additional RAM, over this level, since it includes the cache of + // currently-unnused, re-useable slices, that is not tracked by cacheLimiter. + const gbToUsePerCpu = 0.5 // should be enough to support the amount of traffic 1 CPU can drive, and also less than the typical installed RAM-per-CPU gbToUse := float32(runtime.NumCPU()) * gbToUsePerCpu - if gbToUse > 8 { - gbToUse = 8 // cap it. We don't need more than this. Even 6 is enough at 10 Gbps with standard chunk sizes, but allow a little extra here to help if larger blob block sizes are selected by user + if gbToUse > 10 { + gbToUse = 10 // cap it. Even 6 is enough at 10 Gbps with standard chunk sizes, but allow a little extra here to help if larger blob block sizes are selected by user } maxRamBytesToUse := int64(gbToUse * 1024 * 1024 * 1024) @@ -165,6 +167,9 @@ func initJobsAdmin(appCtx context.Context, concurrentConnections int, targetRate JobsAdmin = ja + // Spin up slice pool pruner + go ja.slicePoolPruneLoop() + // One routine constantly monitors the partsChannel. It takes the JobPartManager from // the Channel and schedules the transfers of that JobPart. go ja.scheduleJobParts() @@ -212,7 +217,7 @@ func (ja *jobsAdmin) chunkProcessor(workerID int) { // We check for suicides first to shrink goroutine pool // Then, we check chunks: normal & low priority select { - case <-ja.xferChannels.suicideCh: // note: as at Dec 2018, this channel is not (yet) used + case <-ja.xferChannels.suicideCh: // note: as at Dec 2018, this channel is not (yet) used return default: select { @@ -224,10 +229,10 @@ func (ja *jobsAdmin) chunkProcessor(workerID int) { chunkFunc(workerID) default: time.Sleep(100 * time.Millisecond) // Sleep before looping around - // TODO: Question: In order to safely support high goroutine counts, - // do we need to review sleep duration, or find an approach that does not require waking every x milliseconds - // For now, duration has been increased substantially from the previous 1 ms, to reduce cost of - // the wake-ups. + // TODO: Question: In order to safely support high goroutine counts, + // do we need to review sleep duration, or find an approach that does not require waking every x milliseconds + // For now, duration has been increased substantially from the previous 1 ms, to reduce cost of + // the wake-ups. } } } @@ -282,7 +287,7 @@ type jobsAdmin struct { xferChannels XferChannels appCtx context.Context pacer *pacer - slicePool common.ByteSlicePooler + slicePool common.ByteSlicePooler cacheLimiter common.CacheLimiter } @@ -478,6 +483,23 @@ func (ja *jobsAdmin) Log(level pipeline.LogLevel, msg string) { ja.logger.Log(le func (ja *jobsAdmin) Panic(err error) { ja.logger.Panic(err) } func (ja *jobsAdmin) CloseLog() { ja.logger.CloseLog() } +func (ja *jobsAdmin) slicePoolPruneLoop() { + // if something in the pool has been unused for this long, we probably don't need it + const pruneInterval = 5 * time.Second + + ticker := time.NewTicker(pruneInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + ja.slicePool.Prune() + case <-ja.appCtx.Done(): + break + } + } +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // The jobIDToJobMgr maps each JobID to its JobMgr From b5aac968211aa416169e9945e09120b9f04d3083 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Thu, 21 Feb 2019 00:42:05 -0800 Subject: [PATCH 42/64] Improved usability of sync command --- cmd/sync.go | 72 +++++++++++++++++++++-------- cmd/syncEnumerator.go | 36 +++++++-------- cmd/syncProcessor.go | 59 +++++++++++++---------- cmd/zt_scenario_helpers_for_test.go | 31 ++++++++++--- cmd/zt_sync_download_test.go | 9 ++-- cmd/zt_sync_processor_test.go | 12 ++--- cmd/zt_sync_upload_test.go | 5 +- common/fe-ste-models.go | 20 ++++++++ common/rpc-models.go | 4 ++ ste/init.go | 10 +++- 10 files changed, 177 insertions(+), 81 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index 3b18144a7..d9b7e39b4 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -53,10 +53,10 @@ type rawSyncCmdArgs struct { followSymlinks bool output string md5ValidationOption string - // this flag predefines the user-agreement to delete the files in case sync found some files at destination - // which doesn't exists at source. With this flag turned on, user will not be asked for permission before - // deleting the flag. - force bool + // this flag indicates the user agreement with respect to deleting the extra files at the destination + // which do not exists at source. With this flag turned on/off, users will not be asked for permission. + // otherwise the user is prompted to make a decision + deleteDestination string } func (raw *rawSyncCmdArgs) parsePatterns(pattern string) (cookedPatterns []string) { @@ -125,13 +125,18 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { cooked.blockSize = raw.blockSize cooked.followSymlinks = raw.followSymlinks cooked.recursive = raw.recursive - cooked.force = raw.force + + // determine whether we should prompt the user to delete extra files + err := cooked.deleteDestination.Parse(raw.deleteDestination) + if err != nil { + return cooked, err + } // parse the filter patterns cooked.include = raw.parsePatterns(raw.include) cooked.exclude = raw.parsePatterns(raw.exclude) - err := cooked.logVerbosity.Parse(raw.logVerbosity) + err = cooked.logVerbosity.Parse(raw.logVerbosity) if err != nil { return cooked, err } @@ -203,10 +208,14 @@ type cookedSyncCmdArgs struct { atomicSourceFilesScanned uint64 // defines the number of files listed at the destination and compared. atomicDestinationFilesScanned uint64 - // this flag determines the user-agreement to delete the files in case sync found files/blobs at the destination - // that do not exist at the source. With this flag turned on, user will not be prompted for permission before - // deleting the files/blobs. - force bool + + // deletion count keeps track of how many extra files from the destination were removed + deletionCount uint32 + + // this flag indicates the user agreement with respect to deleting the extra files at the destination + // which do not exists at source. With this flag turned on/off, users will not be asked for permission. + // otherwise the user is prompted to make a decision + deleteDestination common.DeleteDestination } // setFirstPartOrdered sets the value of atomicFirstPartOrdered to 1 @@ -271,6 +280,11 @@ func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { } } +func (cca *cookedSyncCmdArgs) reportScanningProgress(lcm common.LifecycleMgr, throughputString string) { + lcm.Progress(fmt.Sprintf("%v Files Scanned at Source, %v Files Scanned at Destination%s", + atomic.LoadUint64(&cca.atomicSourceFilesScanned), atomic.LoadUint64(&cca.atomicDestinationFilesScanned), throughputString)) +} + func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { var summary common.ListSyncJobSummaryResponse var throughput float64 @@ -279,7 +293,7 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { // fetch a job status and compute throughput if the first part was dispatched if cca.firstPartOrdered() { Rpc(common.ERpcCmd.ListSyncJobSummary(), &cca.jobID, &summary) - jobDone = summary.JobStatus == common.EJobStatus.Completed() || summary.JobStatus == common.EJobStatus.Cancelled() + jobDone = summary.JobStatus.IsJobDone() // compute the average throughput for the last time interval bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) * 8 / float64(1024*1024)) @@ -304,14 +318,18 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { throughputString = fmt.Sprintf(", 2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) } - lcm.Progress(fmt.Sprintf("%v File Scanned at Source, %v Files Scanned at Destination%s", - atomic.LoadUint64(&cca.atomicSourceFilesScanned), atomic.LoadUint64(&cca.atomicDestinationFilesScanned), throughputString)) + cca.reportScanningProgress(lcm, throughputString) return } // if json output is desired, simply marshal and return // note that if job is already done, we simply exit if cca.output == common.EOutputFormat.Json() { + // TODO figure out if deletions should be done by the enumeration engine or not + // TODO if not, remove this so that we get the proper number from the ste + summary.DeleteTotalTransfers = cca.deletionCount + summary.DeleteTransfersCompleted = cca.deletionCount + //jsonOutput, err := json.MarshalIndent(summary, "", " ") jsonOutput, err := json.Marshal(summary) common.PanicIfErr(err) @@ -336,15 +354,29 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { exitCode = common.EExitCode.Error() } lcm.Exit(fmt.Sprintf( - "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Copy Transfers: %v\nTotal Number Of Delete Transfers: %v\nNumber of Copy Transfers Completed: %v\nNumber of Copy Transfers Failed: %v\nNumber of Delete Transfers Completed: %v\nNumber of Delete Transfers Failed: %v\nFinal Job Status: %v\n", + ` +Job %s Summary +Files Scanned at Source: %v +Files Scanned at Destination: %v +Elapsed Time (Minutes): %v +Total Number Of Copy Transfers: %v +Number of Copy Transfers Completed: %v +Number of Copy Transfers Failed: %v +Number of Deletions at Destination: %v +Total Number of Bytes Transferred: %v +Total Number of Bytes Enumerated: %v +Final Job Status: %v +`, summary.JobID.String(), + cca.atomicSourceFilesScanned, + cca.atomicDestinationFilesScanned, ste.ToFixed(duration.Minutes(), 4), summary.CopyTotalTransfers, - summary.DeleteTotalTransfers, summary.CopyTransfersCompleted, summary.CopyTransfersFailed, - summary.DeleteTransfersCompleted, - summary.DeleteTransfersFailed, + cca.deletionCount, + summary.TotalBytesTransferred, + summary.TotalBytesEnumerated, summary.JobStatus), exitCode) } @@ -457,9 +489,9 @@ func init() { syncCmd.PersistentFlags().StringVar(&raw.include, "include", "", "only include files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") syncCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") - syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "WARNING", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") - syncCmd.PersistentFlags().BoolVar(&raw.force, "force", false, "defines user's decision to delete extra files at the destination that are not present at the source. "+ - "If false, user will be prompted with a question while scheduling files/blobs for deletion.") + syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") + syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "prompt", "defines whether to delete extra files from the destination that are not present at the source. Could be set to true or false."+ + "If not specified, user will be prompted with a question while scheduling files/blobs for deletion.") syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading.") // TODO: should the previous line list the allowable values? diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 6e7a5e6ed..43118af46 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -62,11 +62,6 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat comparator := newSyncSourceFilter(indexer) finalize := func() error { - jobInitiated, err := transferScheduler.dispatchFinalPart() - if err != nil { - return err - } - // remove the extra files at the destination that were not present at the source // we can only know what needs to be deleted when we have FINISHED traversing the remote source // since only then can we know which local files definitely don't exist remotely @@ -76,13 +71,14 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat return err } - if !jobInitiated && !deleteScheduler.wasAnyFileDeleted() { - return errors.New("the source and destination are already in sync") - } else if !jobInitiated && deleteScheduler.wasAnyFileDeleted() { - // some files were deleted but no transfer scheduled - glcm.Exit("the source and destination are now in sync", common.EExitCode.Success()) + // let the deletions happen first + // otherwise if the final part is executed too quickly, we might quit before deletions could finish + jobInitiated, err := transferScheduler.dispatchFinalPart() + if err != nil { + return err } + quitIfInSync(jobInitiated, deleteScheduler.wasAnyFileDeleted(), cca) cca.setScanningComplete() return nil } @@ -138,19 +134,12 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator return err } - anyBlobDeleted := destinationCleaner.wasAnyFileDeleted() jobInitiated, err := transferScheduler.dispatchFinalPart() if err != nil { return err } - if !jobInitiated && !anyBlobDeleted { - return errors.New("the source and destination are already in sync") - } else if !jobInitiated && anyBlobDeleted { - // some files were deleted but no transfer scheduled - glcm.Exit("the source and destination are now in sync", common.EExitCode.Success()) - } - + quitIfInSync(jobInitiated, destinationCleaner.wasAnyFileDeleted(), cca) cca.setScanningComplete() return nil } @@ -158,3 +147,14 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator return newSyncEnumerator(sourceTraverser, destinationTraverser, indexer, filters, comparator, transferScheduler.scheduleCopyTransfer, finalize), nil } + +func quitIfInSync(transferJobInitiated, anyDestinationFileDeleted bool, cca *cookedSyncCmdArgs) { + if !transferJobInitiated && !anyDestinationFileDeleted { + cca.reportScanningProgress(glcm, "") + glcm.Exit("The source and destination are already in sync.", common.EExitCode.Success()) + } else if !transferJobInitiated && anyDestinationFileDeleted { + // some files were deleted but no transfer scheduled + cca.reportScanningProgress(glcm, "") + glcm.Exit("The source and destination are now in sync.", common.EExitCode.Success()) + } +} diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index 16ac9f388..5229264fb 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -67,24 +67,24 @@ type interactiveDeleteProcessor struct { // the plugged-in deleter that performs the actual deletion deleter objectProcessor - // whether force delete is on - force bool + // whether we should ask the user for permission the first time we delete a file + shouldPromptUser bool - // ask the user for permission the first time we delete a file - hasPromptedUser bool + // note down whether any delete should happen + shouldDelete bool // used for prompt message // examples: "blobs", "local files", etc. objectType string - // note down whether any delete should happen - shouldDelete bool + // count the deletions that happened + deletionCount *uint32 } func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err error) { - if !d.hasPromptedUser { - d.shouldDelete = d.promptForConfirmation() - d.hasPromptedUser = true + if d.shouldPromptUser { + d.shouldDelete = d.promptForConfirmation() // note down the user's decision + d.shouldPromptUser = false // only prompt the first time that this function is called } if !d.shouldDelete { @@ -96,35 +96,45 @@ func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err glcm.Info(fmt.Sprintf("error %s deleting the object %s", err.Error(), object.relativePath)) } + if d.deletionCount != nil { + // increment the count since we need to report to user + *d.deletionCount += +1 + } return } func (d *interactiveDeleteProcessor) promptForConfirmation() (shouldDelete bool) { shouldDelete = false - // omit asking if the user has already specified - if d.force { + answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them from the destination? Please confirm with y/n: ", d.objectType)) + if answer == "y" || answer == "yes" { shouldDelete = true + glcm.Info(fmt.Sprintf("Confirmed. The extra %s will be deleted:", d.objectType)) } else { - answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them? Please confirm with y/n: ", d.objectType)) - if answer == "y" || answer == "yes" { - shouldDelete = true - glcm.Info(fmt.Sprintf("Confirmed. The extra %s will be deleted:", d.objectType)) - } else { - glcm.Info("No deletions will happen.") - } + glcm.Info("No deletions will happen.") } return } func (d *interactiveDeleteProcessor) wasAnyFileDeleted() bool { - // we'd have prompted the user if any stored object was passed in - return d.hasPromptedUser + return *d.deletionCount > 0 +} + +func newInteractiveDeleteProcessor(deleter objectProcessor, deleteDestination common.DeleteDestination, + objectType string, deleteCount *uint32) *interactiveDeleteProcessor { + + return &interactiveDeleteProcessor{ + deleter: deleter, + objectType: objectType, + deletionCount: deleteCount, + shouldPromptUser: deleteDestination == common.EDeleteDestination.Prompt(), + shouldDelete: deleteDestination == common.EDeleteDestination.True(), // if shouldPromptUser is true, this will start as false, but we will determine its value later + } } func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs) *interactiveDeleteProcessor { localDeleter := localFileDeleter{rootPath: cca.destination} - return &interactiveDeleteProcessor{deleter: localDeleter.deleteFile, force: cca.force, objectType: "local files"} + return newInteractiveDeleteProcessor(localDeleter.deleteFile, cca.deleteDestination, "local files", &cca.deletionCount) } type localFileDeleter struct { @@ -132,7 +142,7 @@ type localFileDeleter struct { } func (l *localFileDeleter) deleteFile(object storedObject) error { - glcm.Info("Deleting file: " + object.relativePath) + glcm.Info("Deleting extra file: " + object.relativePath) return os.Remove(filepath.Join(l.rootPath, object.relativePath)) } @@ -150,7 +160,8 @@ func newSyncBlobDeleteProcessor(cca *cookedSyncCmdArgs) (*interactiveDeleteProce return nil, err } - return &interactiveDeleteProcessor{deleter: newBlobDeleter(rawURL, p, ctx).deleteBlob, force: cca.force, objectType: "blobs"}, nil + return newInteractiveDeleteProcessor(newBlobDeleter(rawURL, p, ctx).deleteBlob, + cca.deleteDestination, "blobs", &cca.deletionCount), nil } type blobDeleter struct { @@ -168,7 +179,7 @@ func newBlobDeleter(rawRootURL *url.URL, p pipeline.Pipeline, ctx context.Contex } func (b *blobDeleter) deleteBlob(object storedObject) error { - glcm.Info("Deleting: " + object.relativePath) + glcm.Info("Deleting extra blob: " + object.relativePath) // construct the blob URL using its relative path // the rootURL could be pointing to a container, or a virtual directory diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 2ad121d69..8a982975f 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -87,20 +87,39 @@ func (s scenarioHelper) generateFilesFromList(c *chk.C, dirPath string, fileList } } -// make 30 blobs with random names +// make 50 blobs with random names // 10 of them at the top level // 10 of them in sub dir "sub1" // 10 of them in sub dir "sub2" +// 10 of them in deeper sub dir "sub1/sub3/sub5" +// 10 of them with special characters func (scenarioHelper) generateCommonRemoteScenario(c *chk.C, containerURL azblob.ContainerURL, prefix string) (blobList []string) { - blobList = make([]string, 30) + blobList = make([]string, 50) + specialNames := []string{ + "打麻将.txt", + "wow such space so much space", + "saywut.pdf?yo=bla&WUWUWU=foo&sig=yyy", + "coração", + "আপনার নাম কি", + "%4509%4254$85140&", + "Donaudampfschifffahrtselektrizitätenhauptbetriebswerkbauunterbeamtengesellschaft", + "お名前は何ですか", + "Adın ne", + "як вас звати", + } + for i := 0; i < 10; i++ { _, blobName1 := createNewBlockBlob(c, containerURL, prefix+"top") _, blobName2 := createNewBlockBlob(c, containerURL, prefix+"sub1/") _, blobName3 := createNewBlockBlob(c, containerURL, prefix+"sub2/") - - blobList[3*i] = blobName1 - blobList[3*i+1] = blobName2 - blobList[3*i+2] = blobName3 + _, blobName4 := createNewBlockBlob(c, containerURL, prefix+"sub1/sub3/sub5/") + _, blobName5 := createNewBlockBlob(c, containerURL, prefix+specialNames[i]) + + blobList[5*i] = blobName1 + blobList[5*i+1] = blobName2 + blobList[5*i+2] = blobName3 + blobList[5*i+3] = blobName4 + blobList[5*i+4] = blobName5 } return diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index a050adf6b..1252ad841 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -73,13 +73,15 @@ func validateTransfersAreScheduled(c *chk.C, srcDirName, dstDirName string, expe } func getDefaultRawInput(src, dst string) rawSyncCmdArgs { + deleteDestination := common.EDeleteDestination.True() + return rawSyncCmdArgs{ src: src, dst: dst, recursive: true, logVerbosity: defaultLogVerbosityForSync, output: defaultOutputFormatForSync, - force: true, + deleteDestination: deleteDestination.String(), md5ValidationOption: common.DefaultHashValidationOption.String(), } } @@ -112,7 +114,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { // the file was created after the blob, so no sync should happen runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.NotNil) + c.Assert(err, chk.IsNil) // validate that the right number of transfers were scheduled c.Assert(len(mockedRPC.transfers), chk.Equals, 0) @@ -206,7 +208,8 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) raw := getDefaultRawInput(rawContainerURLWithSAS.String(), dstDirName) runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.NotNil) + c.Assert(err, chk.IsNil) + // validate that the right number of transfers were scheduled c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) diff --git a/cmd/zt_sync_processor_test.go b/cmd/zt_sync_processor_test.go index 1dd02ea11..a7ea5964c 100644 --- a/cmd/zt_sync_processor_test.go +++ b/cmd/zt_sync_processor_test.go @@ -41,8 +41,8 @@ func (s *syncProcessorSuite) TestLocalDeleter(c *chk.C) { // construct the cooked input to simulate user input cca := &cookedSyncCmdArgs{ - destination: dstDirName, - force: true, + destination: dstDirName, + deleteDestination: common.EDeleteDestination.True(), } // set up local deleter @@ -79,10 +79,10 @@ func (s *syncProcessorSuite) TestBlobDeleter(c *chk.C) { rawContainerURL := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) parts := azblob.NewBlobURLParts(rawContainerURL) cca := &cookedSyncCmdArgs{ - destination: containerURL.String(), - destinationSAS: parts.SAS.Encode(), - credentialInfo: common.CredentialInfo{CredentialType: common.ECredentialType.Anonymous()}, - force: true, + destination: containerURL.String(), + destinationSAS: parts.SAS.Encode(), + credentialInfo: common.CredentialInfo{CredentialType: common.ECredentialType.Anonymous()}, + deleteDestination: common.EDeleteDestination.True(), } // set up the blob deleter diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go index 2531a2bde..6bc47c96d 100644 --- a/cmd/zt_sync_upload_test.go +++ b/cmd/zt_sync_upload_test.go @@ -61,7 +61,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { // the blob was created after the file, so no sync should happen runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.NotNil) + c.Assert(err, chk.IsNil) // validate that the right number of transfers were scheduled c.Assert(len(mockedRPC.transfers), chk.Equals, 0) @@ -154,7 +154,8 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithIdenticalDestination(c *chk.C) { raw := getDefaultRawInput(srcDirName, rawContainerURLWithSAS.String()) runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.NotNil) + c.Assert(err, chk.IsNil) + // validate that the right number of transfers were scheduled c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index cc751af3a..9a4c0156e 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -98,6 +98,26 @@ type Status uint32 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +type DeleteDestination uint32 + +var EDeleteDestination = DeleteDestination(0) + +func (DeleteDestination) Prompt() DeleteDestination { return DeleteDestination(0) } +func (DeleteDestination) True() DeleteDestination { return DeleteDestination(1) } +func (DeleteDestination) False() DeleteDestination { return DeleteDestination(2) } + +func (dd *DeleteDestination) Parse(s string) error { + val, err := enum.Parse(reflect.TypeOf(dd), s, true) + if err == nil { + *dd = val.(DeleteDestination) + } + return err +} + +func (dd DeleteDestination) String() string { + return enum.StringInt(dd, reflect.TypeOf(dd)) +} + type OutputFormat uint32 var EOutputFormat = OutputFormat(0) diff --git a/common/rpc-models.go b/common/rpc-models.go index 34bf4ad50..57fc69a91 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -157,6 +157,10 @@ type ListSyncJobSummaryResponse struct { FailedTransfers []TransferDetail IsDiskConstrained bool PerfStrings []string + // sum of the size of transfer completed successfully so far. + TotalBytesTransferred uint64 + // sum of the total transfer enumerated so far. + TotalBytesEnumerated uint64 } type ListJobTransfersRequest struct { diff --git a/ste/init.go b/ste/init.go index f05b3b729..5ab761cbd 100644 --- a/ste/init.go +++ b/ste/init.go @@ -494,6 +494,9 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse { * DeleteTransfersCompleted - number of delete transfers failed in the job. * FailedTransfers - list of transfer that failed. */ +// TODO determine if this should be removed, since we currently perform the deletions in the enumeration engine +// TODO if deletions are also done in the backend, then we should keep this & improve it potentially +// TODO deletions and copies can currently be placed in different job parts func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { // getJobPartMapFromJobPartInfoMap gives the map of partNo to JobPartPlanInfo Pointer for a given JobId jm, found := JobsAdmin.JobMgr(jobID) @@ -542,6 +545,8 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { if fromTo == common.EFromTo.LocalBlob() || fromTo == common.EFromTo.BlobLocal() { js.CopyTransfersCompleted++ + js.TotalBytesTransferred += uint64(jppt.SourceSize) + js.TotalBytesEnumerated += uint64(jppt.SourceSize) } if fromTo == common.EFromTo.BlobTrash() { js.DeleteTransfersCompleted++ @@ -552,6 +557,7 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { if fromTo == common.EFromTo.LocalBlob() || fromTo == common.EFromTo.BlobLocal() { js.CopyTransfersFailed++ + js.TotalBytesEnumerated += uint64(jppt.SourceSize) } if fromTo == common.EFromTo.BlobTrash() { js.DeleteTransfersFailed++ @@ -602,8 +608,8 @@ func GetSyncJobSummary(jobID common.JobID) common.ListSyncJobSummaryResponse { if js.JobStatus == common.EJobStatus.Completed() { js.JobStatus = js.JobStatus.EnhanceJobStatusInfo(false, - js.CopyTransfersFailed + js.DeleteTransfersFailed > 0, - js.CopyTransfersCompleted + js.DeleteTransfersCompleted > 0) + js.CopyTransfersFailed+js.DeleteTransfersFailed > 0, + js.CopyTransfersCompleted+js.DeleteTransfersCompleted > 0) } return js From 161ee31833096e5de475e889eb4a8df750ca17e0 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Thu, 21 Feb 2019 12:42:58 -0800 Subject: [PATCH 43/64] Improved timing issue for sync integration tests --- cmd/zt_scenario_helpers_for_test.go | 68 +++++++++++++++++------------ cmd/zt_sync_download_test.go | 7 --- cmd/zt_sync_upload_test.go | 14 ------ 3 files changed, 39 insertions(+), 50 deletions(-) diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 8a982975f..35d9455cc 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -29,12 +29,26 @@ import ( "os" "path/filepath" "strings" + "time" ) const defaultFileSize = 1024 type scenarioHelper struct{} +var specialNames = []string{ + "打麻将.txt", + "wow such space so much space", + "saywut.pdf?yo=bla&WUWUWU=foo&sig=yyy", + "coração", + "আপনার নাম কি", + "%4509%4254$85140&", + "Donaudampfschifffahrtselektrizitätenhauptbetriebswerkbauunterbeamtengesellschaft", + "お名前は何ですか", + "Adın ne", + "як вас звати", +} + func (scenarioHelper) generateLocalDirectory(c *chk.C) (dstDirName string) { dstDirName, err := ioutil.TempDir("", "AzCopySyncDownload") c.Assert(err, chk.IsNil) @@ -58,25 +72,25 @@ func (scenarioHelper) generateFile(filePath string, fileSize int) ([]byte, error } func (s scenarioHelper) generateRandomLocalFiles(c *chk.C, dirPath string, prefix string) (fileList []string) { - fileList = make([]string, 30) + fileList = make([]string, 50) for i := 0; i < 10; i++ { - fileName1 := generateName(prefix + "top") - fileName2 := generateName(prefix + "sub1/") - fileName3 := generateName(prefix + "sub2/") - - fileList[3*i] = fileName1 - fileList[3*i+1] = fileName2 - fileList[3*i+2] = fileName3 - - _, err := s.generateFile(filepath.Join(dirPath, fileName1), defaultFileSize) - c.Assert(err, chk.IsNil) - - _, err = s.generateFile(filepath.Join(dirPath, fileName2), defaultFileSize) - c.Assert(err, chk.IsNil) - - _, err = s.generateFile(filepath.Join(dirPath, fileName3), defaultFileSize) - c.Assert(err, chk.IsNil) + batch := []string{ + generateName(prefix + "top"), + generateName(prefix + "sub1/"), + generateName(prefix + "sub2/"), + generateName(prefix + "sub1/sub3/sub5/"), + generateName(prefix + specialNames[i]), + } + + for j, name := range batch { + fileList[5*i+j] = name + _, err := s.generateFile(filepath.Join(dirPath, name), defaultFileSize) + c.Assert(err, chk.IsNil) + } } + + // sleep a bit so that the files' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) return } @@ -85,6 +99,9 @@ func (s scenarioHelper) generateFilesFromList(c *chk.C, dirPath string, fileList _, err := s.generateFile(filepath.Join(dirPath, fileName), defaultFileSize) c.Assert(err, chk.IsNil) } + + // sleep a bit so that the files' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) } // make 50 blobs with random names @@ -95,18 +112,6 @@ func (s scenarioHelper) generateFilesFromList(c *chk.C, dirPath string, fileList // 10 of them with special characters func (scenarioHelper) generateCommonRemoteScenario(c *chk.C, containerURL azblob.ContainerURL, prefix string) (blobList []string) { blobList = make([]string, 50) - specialNames := []string{ - "打麻将.txt", - "wow such space so much space", - "saywut.pdf?yo=bla&WUWUWU=foo&sig=yyy", - "coração", - "আপনার নাম কি", - "%4509%4254$85140&", - "Donaudampfschifffahrtselektrizitätenhauptbetriebswerkbauunterbeamtengesellschaft", - "お名前は何ですか", - "Adın ne", - "як вас звати", - } for i := 0; i < 10; i++ { _, blobName1 := createNewBlockBlob(c, containerURL, prefix+"top") @@ -122,6 +127,8 @@ func (scenarioHelper) generateCommonRemoteScenario(c *chk.C, containerURL azblob blobList[5*i+4] = blobName5 } + // sleep a bit so that the blobs' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) return } @@ -134,6 +141,9 @@ func (scenarioHelper) generateBlobs(c *chk.C, containerURL azblob.ContainerURL, c.Assert(err, chk.IsNil) c.Assert(cResp.StatusCode(), chk.Equals, 201) } + + // sleep a bit so that the blobs' lmts are guaranteed to be in the past + time.Sleep(time.Millisecond * 1500) } // Golang does not have sets, so we have to use a map to fulfill the same functionality diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 1252ad841..73a74fff0 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -29,7 +29,6 @@ import ( "io/ioutil" "path/filepath" "strings" - "time" ) const ( @@ -120,9 +119,6 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) - // sleep for 1 sec so that the blob's last modified times are guaranteed to be newer - time.Sleep(time.Second) - // recreate the blob to have a later last modified time scenarioHelper{}.generateBlobs(c, containerURL, blobList) mockedRPC.reset() @@ -214,9 +210,6 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) - // wait for 1 second so that the last modified times of the blobs are guaranteed to be newer - time.Sleep(time.Second) - // refresh the blobs' last modified time so that they are newer scenarioHelper{}.generateBlobs(c, containerURL, blobList) mockedRPC.reset() diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go index 6bc47c96d..e836fa51a 100644 --- a/cmd/zt_sync_upload_test.go +++ b/cmd/zt_sync_upload_test.go @@ -27,7 +27,6 @@ import ( chk "gopkg.in/check.v1" "path/filepath" "strings" - "time" ) // regular file->blob sync @@ -40,9 +39,6 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { fileList := []string{srcFileName} scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) - // wait for 1 second so that the last modified time of the blob is guaranteed to be newer - time.Sleep(time.Second) - // set up the destination container with a single blob dstBlobName := srcFileName containerURL, containerName := createNewContainer(c, bsu) @@ -67,9 +63,6 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) - // sleep for 1 sec so the blob's last modified time is older - time.Sleep(time.Second) - // recreate the file to have a later last modified time scenarioHelper{}.generateFilesFromList(c, srcDirName, []string{srcFileName}) mockedRPC.reset() @@ -141,7 +134,6 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithIdenticalDestination(c *chk.C) { defer deleteContainer(c, containerURL) // wait for 1 second so that the last modified times of the blobs are guaranteed to be newer - time.Sleep(time.Second) scenarioHelper{}.generateBlobs(c, containerURL, fileList) // set up interceptor @@ -160,9 +152,6 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithIdenticalDestination(c *chk.C) { c.Assert(len(mockedRPC.transfers), chk.Equals, 0) }) - // wait for 1 second so that the last modified times of the files are guaranteed to be newer - time.Sleep(time.Second) - // refresh the files' last modified time so that they are newer scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) mockedRPC.reset() @@ -181,9 +170,6 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithMismatchedDestination(c *chk.C) srcDirName := scenarioHelper{}.generateLocalDirectory(c) fileList := scenarioHelper{}.generateRandomLocalFiles(c, srcDirName, "") - // wait for 1 second so that the last modified times of the blobs are guaranteed to be newer - time.Sleep(time.Second) - // set up an the container with half of the files, but later lmts // also add some extra blobs that are not present at the source extraBlobs := []string{"extraFile1.pdf, extraFile2.txt"} From 950ea0ce3911f2214f73d566790f7cbe2cdb81c6 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Thu, 21 Feb 2019 15:50:20 -0800 Subject: [PATCH 44/64] Changed default value of delete-destination flag to false --- cmd/sync.go | 4 ++-- cmd/syncProcessor.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index d9b7e39b4..a7f3f77eb 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -490,8 +490,8 @@ func init() { syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") syncCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") - syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "prompt", "defines whether to delete extra files from the destination that are not present at the source. Could be set to true or false."+ - "If not specified, user will be prompted with a question while scheduling files/blobs for deletion.") + syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "false", "defines whether to delete extra files from the destination that are not present at the source. Could be set to true, false, or prompt. "+ + "If set to prompt, user will be asked a question before scheduling files/blobs for deletion.") syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading.") // TODO: should the previous line list the allowable values? diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index 5229264fb..ce7a0f5d5 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -106,7 +106,7 @@ func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err func (d *interactiveDeleteProcessor) promptForConfirmation() (shouldDelete bool) { shouldDelete = false - answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them from the destination? Please confirm with y/n: ", d.objectType)) + answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them from the destination? Please confirm with y/n (default: n): ", d.objectType)) if answer == "y" || answer == "yes" { shouldDelete = true glcm.Info(fmt.Sprintf("Confirmed. The extra %s will be deleted:", d.objectType)) From 7fd58159b167cfb2f52750f5f91086fd65ae3be6 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Thu, 21 Feb 2019 19:41:50 -0800 Subject: [PATCH 45/64] Fixed deletion counter for sync to be atomic --- cmd/sync.go | 20 ++++++++++++++------ cmd/syncEnumerator.go | 4 ++-- cmd/syncFilter.go | 8 ++++---- cmd/syncProcessor.go | 39 ++++++++++++++++++++------------------- common/fe-ste-models.go | 6 +++--- 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index a7f3f77eb..1e2d80c55 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -210,7 +210,7 @@ type cookedSyncCmdArgs struct { atomicDestinationFilesScanned uint64 // deletion count keeps track of how many extra files from the destination were removed - deletionCount uint32 + atomicDeletionCount uint32 // this flag indicates the user agreement with respect to deleting the extra files at the destination // which do not exists at source. With this flag turned on/off, users will not be asked for permission. @@ -218,6 +218,14 @@ type cookedSyncCmdArgs struct { deleteDestination common.DeleteDestination } +func (cca *cookedSyncCmdArgs) incrementDeletionCount() { + atomic.AddUint32(&cca.atomicDeletionCount, 1) +} + +func (cca *cookedSyncCmdArgs) getDeletionCount() uint32 { + return atomic.LoadUint32(&cca.atomicDeletionCount) +} + // setFirstPartOrdered sets the value of atomicFirstPartOrdered to 1 func (cca *cookedSyncCmdArgs) setFirstPartOrdered() { atomic.StoreUint32(&cca.atomicFirstPartOrdered, 1) @@ -327,8 +335,8 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { if cca.output == common.EOutputFormat.Json() { // TODO figure out if deletions should be done by the enumeration engine or not // TODO if not, remove this so that we get the proper number from the ste - summary.DeleteTotalTransfers = cca.deletionCount - summary.DeleteTransfersCompleted = cca.deletionCount + summary.DeleteTotalTransfers = cca.getDeletionCount() + summary.DeleteTransfersCompleted = cca.getDeletionCount() //jsonOutput, err := json.MarshalIndent(summary, "", " ") jsonOutput, err := json.Marshal(summary) @@ -368,13 +376,13 @@ Total Number of Bytes Enumerated: %v Final Job Status: %v `, summary.JobID.String(), - cca.atomicSourceFilesScanned, - cca.atomicDestinationFilesScanned, + atomic.LoadUint64(&cca.atomicSourceFilesScanned), + atomic.LoadUint64(&cca.atomicDestinationFilesScanned), ste.ToFixed(duration.Minutes(), 4), summary.CopyTotalTransfers, summary.CopyTransfersCompleted, summary.CopyTransfersFailed, - cca.deletionCount, + cca.atomicDeletionCount, summary.TotalBytesTransferred, summary.TotalBytesEnumerated, summary.JobStatus), exitCode) diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 43118af46..94ae67e1f 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -78,7 +78,7 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat return err } - quitIfInSync(jobInitiated, deleteScheduler.wasAnyFileDeleted(), cca) + quitIfInSync(jobInitiated, cca.getDeletionCount() > 0, cca) cca.setScanningComplete() return nil } @@ -139,7 +139,7 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator return err } - quitIfInSync(jobInitiated, destinationCleaner.wasAnyFileDeleted(), cca) + quitIfInSync(jobInitiated, cca.getDeletionCount() > 0, cca) cca.setScanningComplete() return nil } diff --git a/cmd/syncFilter.go b/cmd/syncFilter.go index 149a16912..9071e8a1f 100644 --- a/cmd/syncFilter.go +++ b/cmd/syncFilter.go @@ -41,13 +41,13 @@ func newSyncDestinationFilter(i *objectIndexer, recyclers objectProcessor) objec // if file x from the destination exists at the source, then we'd only transfer it if it is considered stale compared to its counterpart at the source // if file x does not exist at the source, then it is considered extra, and will be deleted func (f *syncDestinationFilter) doesPass(destinationObject storedObject) bool { - storedObjectInMap, present := f.sourceIndex.indexMap[destinationObject.relativePath] + sourceObjectInMap, present := f.sourceIndex.indexMap[destinationObject.relativePath] // if the destinationObject is present and stale, we let it pass if present { defer delete(f.sourceIndex.indexMap, destinationObject.relativePath) - if storedObjectInMap.isMoreRecentThan(destinationObject) { + if sourceObjectInMap.isMoreRecentThan(destinationObject) { return true } } else { @@ -78,13 +78,13 @@ func newSyncSourceFilter(i *objectIndexer) objectFilter { // note: we remove the storedObject if it is present so that when we have finished // the index will contain all objects which exist at the destination but were NOT passed to this routine func (f *syncSourceFilter) doesPass(sourceObject storedObject) bool { - storedObjectInMap, present := f.destinationIndex.indexMap[sourceObject.relativePath] + destinationObjectInMap, present := f.destinationIndex.indexMap[sourceObject.relativePath] // if the sourceObject is more recent, we let it pass if present { defer delete(f.destinationIndex.indexMap, sourceObject.relativePath) - if sourceObject.isMoreRecentThan(storedObjectInMap) { + if sourceObject.isMoreRecentThan(destinationObjectInMap) { return true } diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index ce7a0f5d5..1c7a3ca0a 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -75,10 +75,14 @@ type interactiveDeleteProcessor struct { // used for prompt message // examples: "blobs", "local files", etc. - objectType string + objectTypeToDisplay string + + // used for prompt message + // examples: a directory path, or url to container + objectLocationToDisplay string // count the deletions that happened - deletionCount *uint32 + incrementDeletionCount func() } func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err error) { @@ -96,9 +100,8 @@ func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err glcm.Info(fmt.Sprintf("error %s deleting the object %s", err.Error(), object.relativePath)) } - if d.deletionCount != nil { - // increment the count since we need to report to user - *d.deletionCount += +1 + if d.incrementDeletionCount != nil { + d.incrementDeletionCount() } return } @@ -106,35 +109,33 @@ func (d *interactiveDeleteProcessor) removeImmediately(object storedObject) (err func (d *interactiveDeleteProcessor) promptForConfirmation() (shouldDelete bool) { shouldDelete = false - answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them from the destination? Please confirm with y/n (default: n): ", d.objectType)) + answer := glcm.Prompt(fmt.Sprintf("Sync has discovered %s that are not present at the source, would you like to delete them from the destination(%s)? Please confirm with y/n (default: n): ", + d.objectTypeToDisplay, d.objectLocationToDisplay)) if answer == "y" || answer == "yes" { shouldDelete = true - glcm.Info(fmt.Sprintf("Confirmed. The extra %s will be deleted:", d.objectType)) + glcm.Info(fmt.Sprintf("Confirmed. The extra %s will be deleted:", d.objectTypeToDisplay)) } else { glcm.Info("No deletions will happen.") } return } -func (d *interactiveDeleteProcessor) wasAnyFileDeleted() bool { - return *d.deletionCount > 0 -} - func newInteractiveDeleteProcessor(deleter objectProcessor, deleteDestination common.DeleteDestination, - objectType string, deleteCount *uint32) *interactiveDeleteProcessor { + objectTypeToDisplay string, objectLocationToDisplay string, incrementDeletionCounter func()) *interactiveDeleteProcessor { return &interactiveDeleteProcessor{ - deleter: deleter, - objectType: objectType, - deletionCount: deleteCount, - shouldPromptUser: deleteDestination == common.EDeleteDestination.Prompt(), - shouldDelete: deleteDestination == common.EDeleteDestination.True(), // if shouldPromptUser is true, this will start as false, but we will determine its value later + deleter: deleter, + objectTypeToDisplay: objectTypeToDisplay, + objectLocationToDisplay: objectLocationToDisplay, + incrementDeletionCount: incrementDeletionCounter, + shouldPromptUser: deleteDestination == common.EDeleteDestination.Prompt(), + shouldDelete: deleteDestination == common.EDeleteDestination.True(), // if shouldPromptUser is true, this will start as false, but we will determine its value later } } func newSyncLocalDeleteProcessor(cca *cookedSyncCmdArgs) *interactiveDeleteProcessor { localDeleter := localFileDeleter{rootPath: cca.destination} - return newInteractiveDeleteProcessor(localDeleter.deleteFile, cca.deleteDestination, "local files", &cca.deletionCount) + return newInteractiveDeleteProcessor(localDeleter.deleteFile, cca.deleteDestination, "local files", cca.destination, cca.incrementDeletionCount) } type localFileDeleter struct { @@ -161,7 +162,7 @@ func newSyncBlobDeleteProcessor(cca *cookedSyncCmdArgs) (*interactiveDeleteProce } return newInteractiveDeleteProcessor(newBlobDeleter(rawURL, p, ctx).deleteBlob, - cca.deleteDestination, "blobs", &cca.deletionCount), nil + cca.deleteDestination, "blobs", cca.destination, cca.incrementDeletionCount), nil } type blobDeleter struct { diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 9a4c0156e..714191c16 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -102,9 +102,9 @@ type DeleteDestination uint32 var EDeleteDestination = DeleteDestination(0) -func (DeleteDestination) Prompt() DeleteDestination { return DeleteDestination(0) } -func (DeleteDestination) True() DeleteDestination { return DeleteDestination(1) } -func (DeleteDestination) False() DeleteDestination { return DeleteDestination(2) } +func (DeleteDestination) False() DeleteDestination { return DeleteDestination(0) } +func (DeleteDestination) Prompt() DeleteDestination { return DeleteDestination(1) } +func (DeleteDestination) True() DeleteDestination { return DeleteDestination(2) } func (dd *DeleteDestination) Parse(s string) error { val, err := enum.Parse(reflect.TypeOf(dd), s, true) From 498fd65a801214a82484a36f15949e48b01e2bbf Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 22 Feb 2019 00:51:53 -0800 Subject: [PATCH 46/64] Uncomment sync test that was previously flaky --- cmd/zt_sync_download_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 73a74fff0..da8a8a62d 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -178,7 +178,6 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithEmptyDestination(c *chk.C) { }) } -/* commented out. Ze will put it back in in his next change // regular container->directory sync but destination is identical to the source, transfers are scheduled based on lmt func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) { bsu := getBSU() @@ -219,7 +218,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) }) } -*/ + // regular container->directory sync where destination is missing some files from source, and also has some extra files func (s *cmdIntegrationSuite) TestSyncDownloadWithMismatchedDestination(c *chk.C) { bsu := getBSU() From 49c18c337c8a917a823e97a2ba3fb3aff53b8b95 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 22 Feb 2019 17:47:34 -0800 Subject: [PATCH 47/64] Refactor sync filters into comparators When the destination is traversed second(*), it is impossible to preserve source properties if everything is just done by the destination traverser. (Because it doesn't see source objects). So in this commit we move the final processing step into a comparator. Because it can see both objects, it can use the properties of both to correctly schedule the right operation - including preservation of source properties --- cmd/{syncFilter.go => syncComparator.go} | 53 +++++---- cmd/syncEnumerator.go | 12 +- cmd/zc_enumerator.go | 35 +++--- cmd/zt_sync_comparator_test.go | 139 +++++++++++++++++++++++ cmd/zt_sync_filter_test.go | 97 ---------------- 5 files changed, 192 insertions(+), 144 deletions(-) rename cmd/{syncFilter.go => syncComparator.go} (63%) create mode 100644 cmd/zt_sync_comparator_test.go delete mode 100644 cmd/zt_sync_filter_test.go diff --git a/cmd/syncFilter.go b/cmd/syncComparator.go similarity index 63% rename from cmd/syncFilter.go rename to cmd/syncComparator.go index 9071e8a1f..173043046 100644 --- a/cmd/syncFilter.go +++ b/cmd/syncComparator.go @@ -21,34 +21,40 @@ package cmd // with the help of an objectIndexer containing the source objects -// filter out the destination objects that should be transferred +// find out the destination objects that should be transferred // in other words, this should be used when destination is being enumerated secondly -type syncDestinationFilter struct { +type syncDestinationComparator struct { // the rejected objects would be passed to the destinationCleaner destinationCleaner objectProcessor + // the processor responsible for scheduling copy transfers + copyTransferScheduler objectProcessor + // storing the source objects sourceIndex *objectIndexer } -func newSyncDestinationFilter(i *objectIndexer, recyclers objectProcessor) objectFilter { - return &syncDestinationFilter{sourceIndex: i, destinationCleaner: recyclers} +func newSyncDestinationComparator(i *objectIndexer, copyScheduler, cleaner objectProcessor) *syncDestinationComparator { + return &syncDestinationComparator{sourceIndex: i, copyTransferScheduler: copyScheduler, destinationCleaner: cleaner} } -// it will only pass destination objects that are present in the indexer but stale compared to the entry in the map -// if the destinationObject is not present at all, it will be passed to the destinationCleaner +// it will only schedule transfers for destination objects that are present in the indexer but stale compared to the entry in the map +// if the destinationObject is not at the source, it will be passed to the destinationCleaner // ex: we already know what the source contains, now we are looking at objects at the destination // if file x from the destination exists at the source, then we'd only transfer it if it is considered stale compared to its counterpart at the source // if file x does not exist at the source, then it is considered extra, and will be deleted -func (f *syncDestinationFilter) doesPass(destinationObject storedObject) bool { +func (f *syncDestinationComparator) processIfNecessary(destinationObject storedObject) error { sourceObjectInMap, present := f.sourceIndex.indexMap[destinationObject.relativePath] - // if the destinationObject is present and stale, we let it pass + // if the destinationObject is present at source and stale, we transfer the up-to-date version from source if present { defer delete(f.sourceIndex.indexMap, destinationObject.relativePath) if sourceObjectInMap.isMoreRecentThan(destinationObject) { - return true + err := f.copyTransferScheduler(sourceObjectInMap) + if err != nil { + return err + } } } else { // purposefully ignore the error from destinationCleaner @@ -56,40 +62,45 @@ func (f *syncDestinationFilter) doesPass(destinationObject storedObject) bool { _ = f.destinationCleaner(destinationObject) } - return false + return nil } // with the help of an objectIndexer containing the destination objects // filter out the source objects that should be transferred // in other words, this should be used when source is being enumerated secondly -type syncSourceFilter struct { +type syncSourceComparator struct { + // the processor responsible for scheduling copy transfers + copyTransferScheduler objectProcessor // storing the destination objects destinationIndex *objectIndexer } -func newSyncSourceFilter(i *objectIndexer) objectFilter { - return &syncSourceFilter{destinationIndex: i} +func newSyncSourceComparator(i *objectIndexer, copyScheduler objectProcessor) *syncSourceComparator { + return &syncSourceComparator{destinationIndex: i, copyTransferScheduler: copyScheduler} } -// it will only pass items that are: +// it will only transfer source items that are: // 1. not present in the map // 2. present but is more recent than the entry in the map // note: we remove the storedObject if it is present so that when we have finished -// the index will contain all objects which exist at the destination but were NOT passed to this routine -func (f *syncSourceFilter) doesPass(sourceObject storedObject) bool { +// the index will contain all objects which exist at the destination but were NOT seen at the source +func (f *syncSourceComparator) processIfNecessary(sourceObject storedObject) error { destinationObjectInMap, present := f.destinationIndex.indexMap[sourceObject.relativePath] - // if the sourceObject is more recent, we let it pass if present { defer delete(f.destinationIndex.indexMap, sourceObject.relativePath) + // if destination is stale, schedule source for transfer if sourceObject.isMoreRecentThan(destinationObjectInMap) { - return true - } + return f.copyTransferScheduler(sourceObject) - return false + } else { + // skip if source is more recent + return nil + } } - return true + // if source does not exist at the destination, then schedule it for transfer + return f.copyTransferScheduler(sourceObject) } diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 94ae67e1f..abb649042 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -59,7 +59,7 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat // set up the comparator so that the source/destination can be compared indexer := newObjectIndexer() - comparator := newSyncSourceFilter(indexer) + comparator := newSyncSourceComparator(indexer, transferScheduler.scheduleCopyTransfer) finalize := func() error { // remove the extra files at the destination that were not present at the source @@ -83,8 +83,8 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat return nil } - return newSyncEnumerator(destinationTraverser, sourceTraverser, indexer, filters, comparator, - transferScheduler.scheduleCopyTransfer, finalize), nil + return newSyncEnumerator(destinationTraverser, sourceTraverser, indexer, filters, + comparator.processIfNecessary, finalize), nil } // upload implies transferring from a local disk to a remote resource @@ -125,7 +125,7 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator // when uploading, we can delete remote objects immediately, because as we traverse the remote location // we ALREADY have available a complete map of everything that exists locally // so as soon as we see a remote destination object we can know whether it exists in the local source - comparator := newSyncDestinationFilter(indexer, destinationCleaner.removeImmediately) + comparator := newSyncDestinationComparator(indexer, transferScheduler.scheduleCopyTransfer, destinationCleaner.removeImmediately) finalize := func() error { // schedule every local file that doesn't exist at the destination @@ -144,8 +144,8 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator return nil } - return newSyncEnumerator(sourceTraverser, destinationTraverser, indexer, filters, comparator, - transferScheduler.scheduleCopyTransfer, finalize), nil + return newSyncEnumerator(sourceTraverser, destinationTraverser, indexer, filters, + comparator.processIfNecessary, finalize), nil } func quitIfInSync(transferJobInitiated, anyDestinationFileDeleted bool, cca *cookedSyncCmdArgs) { diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index 4a2516e2d..e0ea3a2e9 100644 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -89,27 +89,24 @@ type syncEnumerator struct { // general filters apply to both the primary and secondary traverser filters []objectFilter - // a special filters that apply only to the secondary traverser - // it filters objects as scanning happens, based on the data from the primary traverser stored in the objectIndexer - objectComparator objectFilter - - // the processor responsible for scheduling copy transfers - copyTransferScheduler objectProcessor + // the processor that apply only to the secondary traverser + // it processes objects as scanning happens + // based on the data from the primary traverser stored in the objectIndexer + objectComparator objectProcessor // a finalizer that is always called if the enumeration finishes properly finalize func() error } func newSyncEnumerator(primaryTraverser, secondaryTraverser resourceTraverser, indexer *objectIndexer, - filters []objectFilter, comparator objectFilter, copyTransferScheduler objectProcessor, finalize func() error) *syncEnumerator { + filters []objectFilter, comparator objectProcessor, finalize func() error) *syncEnumerator { return &syncEnumerator{ - primaryTraverser: primaryTraverser, - secondaryTraverser: secondaryTraverser, - objectIndexer: indexer, - filters: filters, - objectComparator: comparator, - copyTransferScheduler: copyTransferScheduler, - finalize: finalize, + primaryTraverser: primaryTraverser, + secondaryTraverser: secondaryTraverser, + objectIndexer: indexer, + filters: filters, + objectComparator: comparator, + finalize: finalize, } } @@ -120,13 +117,11 @@ func (e *syncEnumerator) enumerate() (err error) { return } - // add the objectComparator as an extra filter to the list - // so that it can filter given objects based on what's already indexed - e.filters = append(e.filters, e.objectComparator) - // enumerate the secondary resource and as the objects pass the filters - // they will be scheduled so that transferring can start while scanning is ongoing - err = e.secondaryTraverser.traverse(e.copyTransferScheduler, e.filters) + // they will be passed to the object comparator + // which can process given objects based on what's already indexed + // note: transferring can start while scanning is ongoing + err = e.secondaryTraverser.traverse(e.objectComparator, e.filters) if err != nil { return } diff --git a/cmd/zt_sync_comparator_test.go b/cmd/zt_sync_comparator_test.go new file mode 100644 index 000000000..3a8a869aa --- /dev/null +++ b/cmd/zt_sync_comparator_test.go @@ -0,0 +1,139 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + chk "gopkg.in/check.v1" + "time" +) + +type syncComparatorSuite struct{} + +var _ = chk.Suite(&syncComparatorSuite{}) + +func (s *syncComparatorSuite) TestSyncSourceComparator(c *chk.C) { + dummyCopyScheduler := dummyProcessor{} + srcMD5 := []byte{'s'} + destMD5 := []byte{'d'} + + // set up the indexer as well as the source comparator + indexer := newObjectIndexer() + sourceComparator := newSyncSourceComparator(indexer, dummyCopyScheduler.process) + + // create a sample destination object + sampleDestinationObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now(), md5: destMD5} + + // test the comparator in case a given source object is not present at the destination + // meaning no entry in the index, so the comparator should pass the given object to schedule a transfer + compareErr := sourceComparator.processIfNecessary(storedObject{name: "only_at_source", relativePath: "only_at_source", lastModifiedTime: time.Now(), md5: srcMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // check the source object was indeed scheduled + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 1) + c.Assert(dummyCopyScheduler.record[0].md5, chk.DeepEquals, srcMD5) + + // reset the processor so that it's empty + dummyCopyScheduler = dummyProcessor{} + + // test the comparator in case a given source object is present at the destination + // and it has a later modified time, so the comparator should pass the give object to schedule a transfer + err := indexer.store(sampleDestinationObject) + c.Assert(err, chk.IsNil) + compareErr = sourceComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(time.Hour), md5: srcMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // check the source object was indeed scheduled + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 1) + c.Assert(dummyCopyScheduler.record[0].md5, chk.DeepEquals, srcMD5) + c.Assert(len(indexer.indexMap), chk.Equals, 0) + + // reset the processor so that it's empty + dummyCopyScheduler = dummyProcessor{} + + // test the comparator in case a given source object is present at the destination + // but is has an earlier modified time compared to the one at the destination + // meaning that the source object is considered stale, so no transfer should be scheduled + err = indexer.store(sampleDestinationObject) + c.Assert(err, chk.IsNil) + compareErr = sourceComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour), md5: srcMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // check no source object was scheduled + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 0) + c.Assert(len(indexer.indexMap), chk.Equals, 0) +} + +func (s *syncComparatorSuite) TestSyncDestinationComparator(c *chk.C) { + dummyCopyScheduler := dummyProcessor{} + dummyCleaner := dummyProcessor{} + srcMD5 := []byte{'s'} + destMD5 := []byte{'d'} + + // set up the indexer as well as the destination comparator + indexer := newObjectIndexer() + destinationComparator := newSyncDestinationComparator(indexer, dummyCopyScheduler.process, dummyCleaner.process) + + // create a sample source object + sampleSourceObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now(), md5: srcMD5} + + // test the comparator in case a given destination object is not present at the source + // meaning it is an extra file that needs to be deleted, so the comparator should pass the given object to the destinationCleaner + compareErr := destinationComparator.processIfNecessary(storedObject{name: "only_at_dst", relativePath: "only_at_dst", lastModifiedTime: time.Now(), md5: destMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // verify that destination object is being deleted + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 0) + c.Assert(len(dummyCleaner.record), chk.Equals, 1) + c.Assert(dummyCleaner.record[0].md5, chk.DeepEquals, destMD5) + + // reset dummy processors + dummyCopyScheduler = dummyProcessor{} + dummyCleaner = dummyProcessor{} + + // test the comparator in case a given destination object is present at the source + // and it has a later modified time, since the source data is stale, + // no transfer happens + err := indexer.store(sampleSourceObject) + c.Assert(err, chk.IsNil) + compareErr = destinationComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(time.Hour), md5: destMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // verify that the source object is scheduled for transfer + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 0) + c.Assert(len(dummyCleaner.record), chk.Equals, 0) + + // reset dummy processors + dummyCopyScheduler = dummyProcessor{} + dummyCleaner = dummyProcessor{} + + // test the comparator in case a given destination object is present at the source + // but is has an earlier modified time compared to the one at the source + // meaning that the source object should be transferred since the destination object is stale + err = indexer.store(sampleSourceObject) + c.Assert(err, chk.IsNil) + compareErr = destinationComparator.processIfNecessary(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour), md5: destMD5}) + c.Assert(compareErr, chk.Equals, nil) + + // verify that there's no transfer & no deletes + c.Assert(len(dummyCopyScheduler.record), chk.Equals, 1) + c.Assert(dummyCopyScheduler.record[0].md5, chk.DeepEquals, srcMD5) + c.Assert(len(dummyCleaner.record), chk.Equals, 0) +} diff --git a/cmd/zt_sync_filter_test.go b/cmd/zt_sync_filter_test.go deleted file mode 100644 index e6890fb7d..000000000 --- a/cmd/zt_sync_filter_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright © 2017 Microsoft -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package cmd - -import ( - chk "gopkg.in/check.v1" - "time" -) - -type syncFilterSuite struct{} - -var _ = chk.Suite(&syncFilterSuite{}) - -func (s *syncFilterSuite) TestSyncSourceFilter(c *chk.C) { - // set up the indexer as well as the source filter - indexer := newObjectIndexer() - sourceFilter := newSyncSourceFilter(indexer) - - // create a sample destination object - sampleDestinationObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()} - - // test the filter in case a given source object is not present at the destination - // meaning no entry in the index, so the filter should pass the given object to schedule a transfer - passed := sourceFilter.doesPass(storedObject{name: "only_at_source", relativePath: "only_at_source", lastModifiedTime: time.Now()}) - c.Assert(passed, chk.Equals, true) - - // test the filter in case a given source object is present at the destination - // and it has a later modified time, so the filter should pass the give object to schedule a transfer - err := indexer.store(sampleDestinationObject) - c.Assert(err, chk.IsNil) - passed = sourceFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()}) - c.Assert(passed, chk.Equals, true) - - // test the filter in case a given source object is present at the destination - // but is has an earlier modified time compared to the one at the destination - // meaning that the source object is considered stale, so no transfer should be scheduled - err = indexer.store(sampleDestinationObject) - c.Assert(err, chk.IsNil) - passed = sourceFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour)}) - c.Assert(passed, chk.Equals, false) -} - -func (s *syncFilterSuite) TestSyncDestinationFilter(c *chk.C) { - // set up the indexer as well as the destination filter - indexer := newObjectIndexer() - dummyProcessor := dummyProcessor{} - destinationFilter := newSyncDestinationFilter(indexer, dummyProcessor.process) - - // create a sample source object - sampleSourceObject := storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()} - - // test the filter in case a given destination object is not present at the source - // meaning it is an extra file that needs to be deleted, so the filter should pass the given object to the destinationCleaner - passed := destinationFilter.doesPass(storedObject{name: "only_at_source", relativePath: "only_at_source", lastModifiedTime: time.Now()}) - c.Assert(passed, chk.Equals, false) - c.Assert(len(dummyProcessor.record), chk.Equals, 1) - c.Assert(dummyProcessor.record[0].name, chk.Equals, "only_at_source") - - // reset dummy processor - dummyProcessor.record = make([]storedObject, 0) - - // test the filter in case a given destination object is present at the source - // and it has a later modified time, since the source data is stale, - // the filter should pass not the give object to schedule a transfer - err := indexer.store(sampleSourceObject) - c.Assert(err, chk.IsNil) - passed = destinationFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now()}) - c.Assert(passed, chk.Equals, false) - c.Assert(len(dummyProcessor.record), chk.Equals, 0) - - // test the filter in case a given destination object is present at the source - // but is has an earlier modified time compared to the one at the source - // meaning that the source object should be transferred since the destination object is stale - err = indexer.store(sampleSourceObject) - c.Assert(err, chk.IsNil) - passed = destinationFilter.doesPass(storedObject{name: "test", relativePath: "/usr/test", lastModifiedTime: time.Now().Add(-time.Hour)}) - c.Assert(passed, chk.Equals, true) - c.Assert(len(dummyProcessor.record), chk.Equals, 0) -} From aefef7566acf629a5905dd7b245ef7dc238269be Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Thu, 21 Feb 2019 15:07:11 -0800 Subject: [PATCH 48/64] Added system specific line endings to logger --- common/logger.go | 12 +++++++++--- common/mmf_darwin.go | 2 ++ common/mmf_linux.go | 2 ++ common/mmf_windows.go | 2 ++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common/logger.go b/common/logger.go index 6df504721..eeba30175 100644 --- a/common/logger.go +++ b/common/logger.go @@ -25,9 +25,9 @@ import ( "log" "net/url" "os" - "runtime" - "path" + "runtime" + "strings" "github.com/Azure/azure-pipeline-go/pipeline" ) @@ -131,7 +131,7 @@ func (jl *jobLogger) OpenLog() { jl.file = file jl.logger = log.New(jl.file, "", log.LstdFlags|log.LUTC) // Log the Azcopy Version - jl.logger.Println("AzcopVersion ", AzcopyVersion) + jl.logger.Println("AzcopyVersion ", AzcopyVersion) // Log the OS Environment and OS Architecture jl.logger.Println("OS-Environment ", runtime.GOOS) jl.logger.Println("OS-Architecture ", runtime.GOARCH) @@ -157,6 +157,12 @@ func (jl *jobLogger) CloseLog() { func (jl jobLogger) Log(loglevel pipeline.LogLevel, msg string) { // If the logger for Job is not initialized i.e file is not open // or logger instance is not initialized, then initialize it + + // Go, and therefore the sdk, defaults to \n for line endings, so if the platform has a different line ending, + // we should replace them to ensure readability on the given platform. + if lineEnding != "\n" { + msg = strings.Replace(msg, "\n", lineEnding, -1) + } if jl.ShouldLog(loglevel) { jl.logger.Println(msg) } diff --git a/common/mmf_darwin.go b/common/mmf_darwin.go index 0d4b908af..522351985 100644 --- a/common/mmf_darwin.go +++ b/common/mmf_darwin.go @@ -28,6 +28,8 @@ import ( "syscall" ) +const lineEnding = "\n" + type MMF struct { // slice represents the actual memory mapped buffer slice []byte diff --git a/common/mmf_linux.go b/common/mmf_linux.go index 6dddd39c7..911e34506 100644 --- a/common/mmf_linux.go +++ b/common/mmf_linux.go @@ -28,6 +28,8 @@ import ( "syscall" ) +const lineEnding = "\n" + type MMF struct { // slice represents the actual memory mapped buffer slice []byte diff --git a/common/mmf_windows.go b/common/mmf_windows.go index 89d7b9b4a..bf1192a4e 100644 --- a/common/mmf_windows.go +++ b/common/mmf_windows.go @@ -28,6 +28,8 @@ import ( "unsafe" ) +const lineEnding = "\r\n" + type MMF struct { // slice represents the actual memory mapped buffer slice []byte From bf59025a6a85091c978604c1903cd26c2b2a5d39 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Mon, 25 Feb 2019 17:33:06 -0800 Subject: [PATCH 49/64] Fixed encoding bug in sync command --- cmd/syncProcessor.go | 6 ++- cmd/zc_processor.go | 37 ++++++++++++---- cmd/zt_generic_processor_test.go | 8 ++-- cmd/zt_scenario_helpers_for_test.go | 66 +++++++++++++++++++++++++++++ cmd/zt_sync_download_test.go | 63 +++------------------------ cmd/zt_sync_upload_test.go | 14 +++--- common/fe-ste-models.go | 4 ++ 7 files changed, 121 insertions(+), 77 deletions(-) diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index 1c7a3ca0a..a6d511a3e 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -57,9 +57,13 @@ func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) reportFirstPart := func() { cca.setFirstPartOrdered() } reportFinalPart := func() { cca.isEnumerationComplete = true } + shouldEncodeSource := cca.fromTo.From().IsRemote() + shouldEncodeDestination := cca.fromTo.To().IsRemote() + // note that the source and destination, along with the template are given to the generic processor's constructor // this means that given an object with a relative path, this processor already knows how to schedule the right kind of transfers - return newCopyTransferProcessor(copyJobTemplate, numOfTransfersPerPart, cca.source, cca.destination, reportFirstPart, reportFinalPart) + return newCopyTransferProcessor(copyJobTemplate, numOfTransfersPerPart, cca.source, cca.destination, + shouldEncodeSource, shouldEncodeDestination, reportFirstPart, reportFinalPart) } // base for delete processors targeting different resources diff --git a/cmd/zc_processor.go b/cmd/zc_processor.go index b5da778b3..a0ce08dd4 100644 --- a/cmd/zc_processor.go +++ b/cmd/zc_processor.go @@ -23,6 +23,7 @@ package cmd import ( "fmt" "github.com/Azure/azure-storage-azcopy/common" + "net/url" "strings" ) @@ -32,20 +33,27 @@ type copyTransferProcessor struct { source string destination string + // specify whether source/destination object names need to be URL encoded before dispatching + shouldEscapeSourceObjectName bool + shouldEscapeDestinationObjectName bool + // handles for progress tracking reportFirstPartDispatched func() reportFinalPartDispatched func() } func newCopyTransferProcessor(copyJobTemplate *common.CopyJobPartOrderRequest, numOfTransfersPerPart int, - source string, destination string, reportFirstPartDispatched func(), reportFinalPartDispatched func()) *copyTransferProcessor { + source string, destination string, shouldEscapeSourceObjectName bool, shouldEscapeDestinationObjectName bool, + reportFirstPartDispatched func(), reportFinalPartDispatched func()) *copyTransferProcessor { return ©TransferProcessor{ - numOfTransfersPerPart: numOfTransfersPerPart, - copyJobTemplate: copyJobTemplate, - source: source, - destination: destination, - reportFirstPartDispatched: reportFirstPartDispatched, - reportFinalPartDispatched: reportFinalPartDispatched, + numOfTransfersPerPart: numOfTransfersPerPart, + copyJobTemplate: copyJobTemplate, + source: source, + destination: destination, + shouldEscapeSourceObjectName: shouldEscapeSourceObjectName, + shouldEscapeDestinationObjectName: shouldEscapeDestinationObjectName, + reportFirstPartDispatched: reportFirstPartDispatched, + reportFinalPartDispatched: reportFinalPartDispatched, } } @@ -61,11 +69,14 @@ func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject storedObject) s.copyJobTemplate.PartNum++ } + sourceObjectRelativePath := s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeSourceObjectName) + destinationObjectRelativePath := s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeDestinationObjectName) + // only append the transfer after we've checked and dispatched a part // so that there is at least one transfer for the final part s.copyJobTemplate.Transfers = append(s.copyJobTemplate.Transfers, common.CopyTransfer{ - Source: s.appendObjectPathToResourcePath(storedObject.relativePath, s.source), - Destination: s.appendObjectPathToResourcePath(storedObject.relativePath, s.destination), + Source: s.appendObjectPathToResourcePath(sourceObjectRelativePath, s.source), + Destination: s.appendObjectPathToResourcePath(destinationObjectRelativePath, s.destination), SourceSize: storedObject.size, LastModifiedTime: storedObject.lastModifiedTime, ContentMD5: storedObject.md5, @@ -73,6 +84,14 @@ func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject storedObject) return nil } +func (s *copyTransferProcessor) escapeIfNecessary(path string, shouldEscape bool) string { + if shouldEscape { + return url.PathEscape(path) + } + + return path +} + func (s *copyTransferProcessor) appendObjectPathToResourcePath(storedObjectPath, parentPath string) string { if storedObjectPath == "" { return parentPath diff --git a/cmd/zt_generic_processor_test.go b/cmd/zt_generic_processor_test.go index 9d072f482..32705eaa5 100644 --- a/cmd/zt_generic_processor_test.go +++ b/cmd/zt_generic_processor_test.go @@ -76,7 +76,7 @@ func (s *genericProcessorSuite) TestCopyTransferProcessorMultipleFiles(c *chk.C) for _, numOfParts := range []int{1, 3} { numOfTransfersPerPart := len(sampleObjects) / numOfParts copyProcessor := newCopyTransferProcessor(processorTestSuiteHelper{}.getCopyJobTemplate(), numOfTransfersPerPart, - containerURL.String(), dstDirName, nil, nil) + containerURL.String(), dstDirName, false, false, nil, nil) // go through the objects and make sure they are processed without error for _, storedObject := range sampleObjects { @@ -93,7 +93,7 @@ func (s *genericProcessorSuite) TestCopyTransferProcessorMultipleFiles(c *chk.C) c.Assert(err, chk.IsNil) // assert the right transfers were scheduled - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, + validateTransfersAreScheduled(c, containerURL.String(), false, dstDirName, false, processorTestSuiteHelper{}.getExpectedTransferFromStoredObjectList(sampleObjects), mockedRPC) mockedRPC.reset() @@ -122,7 +122,7 @@ func (s *genericProcessorSuite) TestCopyTransferProcessorSingleFile(c *chk.C) { // set up the processor copyProcessor := newCopyTransferProcessor(processorTestSuiteHelper{}.getCopyJobTemplate(), 2, - containerURL.NewBlockBlobURL(blobList[0]).String(), filepath.Join(dstDirName, dstFileName), nil, nil) + containerURL.NewBlockBlobURL(blobList[0]).String(), filepath.Join(dstDirName, dstFileName), false, false, nil, nil) // exercise the copy transfer processor storedObject := newStoredObject(blobList[0], "", time.Now(), 0, nil) @@ -137,6 +137,6 @@ func (s *genericProcessorSuite) TestCopyTransferProcessorSingleFile(c *chk.C) { c.Assert(jobInitiated, chk.Equals, true) // assert the right transfers were scheduled - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, + validateTransfersAreScheduled(c, containerURL.String(), false, dstDirName, false, blobList, mockedRPC) } diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index 35d9455cc..d962bda9f 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -22,6 +22,7 @@ package cmd import ( "context" + "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-blob-go/azblob" chk "gopkg.in/check.v1" "io/ioutil" @@ -178,3 +179,68 @@ func (scenarioHelper) blobExists(blobURL azblob.BlobURL) bool { } return false } + +func runSyncAndVerify(c *chk.C, raw rawSyncCmdArgs, verifier func(err error)) { + // the simulated user input should parse properly + cooked, err := raw.cook() + c.Assert(err, chk.IsNil) + + // the enumeration ends when process() returns + err = cooked.process() + + // the err is passed to verified, which knows whether it is expected or not + verifier(err) +} + +func validateUploadTransfersAreScheduled(c *chk.C, srcDirName string, dstDirName string, expectedTransfers []string, mockedRPC interceptor) { + validateTransfersAreScheduled(c, srcDirName, false, dstDirName, true, expectedTransfers, mockedRPC) +} + +func validateDownloadTransfersAreScheduled(c *chk.C, srcDirName string, dstDirName string, expectedTransfers []string, mockedRPC interceptor) { + validateTransfersAreScheduled(c, srcDirName, true, dstDirName, false, expectedTransfers, mockedRPC) +} + +func validateTransfersAreScheduled(c *chk.C, srcDirName string, isSrcEncoded bool, dstDirName string, isDstEncoded bool, expectedTransfers []string, mockedRPC interceptor) { + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, len(expectedTransfers)) + + // validate that the right transfers were sent + lookupMap := scenarioHelper{}.convertListToMap(expectedTransfers) + for _, transfer := range mockedRPC.transfers { + srcRelativeFilePath := strings.Replace(transfer.Source, srcDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + dstRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) + + if isSrcEncoded { + srcRelativeFilePath, _ = url.PathUnescape(srcRelativeFilePath) + } + + if isDstEncoded { + dstRelativeFilePath, _ = url.PathUnescape(dstRelativeFilePath) + } + + // the relative paths should be equal + c.Assert(srcRelativeFilePath, chk.Equals, dstRelativeFilePath) + + // look up the source from the expected transfers, make sure it exists + _, srcExist := lookupMap[dstRelativeFilePath] + c.Assert(srcExist, chk.Equals, true) + + // look up the destination from the expected transfers, make sure it exists + _, dstExist := lookupMap[dstRelativeFilePath] + c.Assert(dstExist, chk.Equals, true) + } +} + +func getDefaultRawInput(src, dst string) rawSyncCmdArgs { + deleteDestination := common.EDeleteDestination.True() + + return rawSyncCmdArgs{ + src: src, + dst: dst, + recursive: true, + logVerbosity: defaultLogVerbosityForSync, + output: defaultOutputFormatForSync, + deleteDestination: deleteDestination.String(), + md5ValidationOption: common.DefaultHashValidationOption.String(), + } +} diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index da8a8a62d..205fd3abc 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -36,55 +36,6 @@ const ( defaultOutputFormatForSync = "text" ) -func runSyncAndVerify(c *chk.C, raw rawSyncCmdArgs, verifier func(err error)) { - // the simulated user input should parse properly - cooked, err := raw.cook() - c.Assert(err, chk.IsNil) - - // the enumeration ends when process() returns - err = cooked.process() - - // the err is passed to verified, which knows whether it is expected or not - verifier(err) -} - -func validateTransfersAreScheduled(c *chk.C, srcDirName, dstDirName string, expectedTransfers []string, mockedRPC interceptor) { - // validate that the right number of transfers were scheduled - c.Assert(len(mockedRPC.transfers), chk.Equals, len(expectedTransfers)) - - // validate that the right transfers were sent - lookupMap := scenarioHelper{}.convertListToMap(expectedTransfers) - for _, transfer := range mockedRPC.transfers { - srcRelativeFilePath := strings.Replace(transfer.Source, srcDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) - dstRelativeFilePath := strings.Replace(transfer.Destination, dstDirName+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1) - - // the relative paths should be equal - c.Assert(srcRelativeFilePath, chk.Equals, dstRelativeFilePath) - - // look up the source from the expected transfers, make sure it exists - _, srcExist := lookupMap[dstRelativeFilePath] - c.Assert(srcExist, chk.Equals, true) - - // look up the destination from the expected transfers, make sure it exists - _, dstExist := lookupMap[dstRelativeFilePath] - c.Assert(dstExist, chk.Equals, true) - } -} - -func getDefaultRawInput(src, dst string) rawSyncCmdArgs { - deleteDestination := common.EDeleteDestination.True() - - return rawSyncCmdArgs{ - src: src, - dst: dst, - recursive: true, - logVerbosity: defaultLogVerbosityForSync, - output: defaultOutputFormatForSync, - deleteDestination: deleteDestination.String(), - md5ValidationOption: common.DefaultHashValidationOption.String(), - } -} - // regular blob->file sync func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { bsu := getBSU() @@ -126,7 +77,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) }) } @@ -160,7 +111,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithEmptyDestination(c *chk.C) { c.Assert(len(mockedRPC.transfers), chk.Equals, len(blobList)) // validate that the right transfers were sent - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) }) // turn off recursive, this time only top blobs should be transferred @@ -215,7 +166,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithIdenticalDestination(c *chk.C) runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) }) } @@ -247,7 +198,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithMismatchedDestination(c *chk.C runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, expectedOutput, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, expectedOutput, mockedRPC) // make sure the extra files were deleted currentDstFileList, err := ioutil.ReadDir(dstDirName) @@ -293,7 +244,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithIncludeFlag(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) }) } @@ -328,7 +279,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithExcludeFlag(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) }) } @@ -370,7 +321,7 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithIncludeAndExcludeFlag(c *chk.C runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobsToInclude, mockedRPC) }) } diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go index e836fa51a..264237257 100644 --- a/cmd/zt_sync_upload_test.go +++ b/cmd/zt_sync_upload_test.go @@ -71,7 +71,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) }) } @@ -103,7 +103,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithEmptyDestination(c *chk.C) { c.Assert(len(mockedRPC.transfers), chk.Equals, len(fileList)) // validate that the right transfers were sent - validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) }) // turn off recursive, this time only top blobs should be transferred @@ -158,7 +158,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithIdenticalDestination(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) }) } @@ -190,7 +190,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithMismatchedDestination(c *chk.C) runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, srcDirName, containerURL.String(), expectedOutput, mockedRPC) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), expectedOutput, mockedRPC) // make sure the extra blobs were deleted for _, blobName := range extraBlobs { @@ -229,7 +229,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithIncludeFlag(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) }) } @@ -262,7 +262,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithExcludeFlag(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) }) } @@ -302,7 +302,7 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithIncludeAndExcludeFlag(c *chk.C) runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), filesToInclude, mockedRPC) }) } diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 714191c16..c676b2642 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -297,6 +297,10 @@ func fromToValue(from Location, to Location) FromTo { return FromTo((FromTo(from) << 8) | FromTo(to)) } +func (l Location) IsRemote() bool { + return l == ELocation.BlobFS() || l == ELocation.Blob() || l == ELocation.File() +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// var EFromTo = FromTo(0) From ee76df15a08b928dc0dcd611e44537e769373aff Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Wed, 27 Feb 2019 14:32:02 -0800 Subject: [PATCH 50/64] Added additional tests for single file sync with special chars --- cmd/zt_sync_download_test.go | 79 ++++++++++++++++++----------------- cmd/zt_sync_upload_test.go | 81 ++++++++++++++++++------------------ common/fe-ste-models.go | 11 ++++- 3 files changed, 91 insertions(+), 80 deletions(-) diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 205fd3abc..4d574c72b 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -40,45 +40,46 @@ const ( func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { bsu := getBSU() - // set up the container with a single blob - blobName := "singleblobisbest" - blobList := []string{blobName} - containerURL, containerName := createNewContainer(c, bsu) - scenarioHelper{}.generateBlobs(c, containerURL, blobList) - defer deleteContainer(c, containerURL) - c.Assert(containerURL, chk.NotNil) - - // set up the destination as a single file - dstDirName := scenarioHelper{}.generateLocalDirectory(c) - dstFileName := blobName - scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) - - // set up interceptor - mockedRPC := interceptor{} - Rpc = mockedRPC.intercept - mockedRPC.init() - - // construct the raw input to simulate user input - rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) - raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) - - // the file was created after the blob, so no sync should happen - runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.IsNil) - - // validate that the right number of transfers were scheduled - c.Assert(len(mockedRPC.transfers), chk.Equals, 0) - }) - - // recreate the blob to have a later last modified time - scenarioHelper{}.generateBlobs(c, containerURL, blobList) - mockedRPC.reset() - - runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.IsNil) - - validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) - }) + for _, blobName := range []string{"singleblobisbest", "打麻将.txt", "%4509%4254$85140&"} { + // set up the container with a single blob + blobList := []string{blobName} + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up the destination as a single file + dstDirName := scenarioHelper{}.generateLocalDirectory(c) + dstFileName := blobName + scenarioHelper{}.generateFilesFromList(c, dstDirName, blobList) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) + raw := getDefaultRawInput(rawBlobURLWithSAS.String(), filepath.Join(dstDirName, dstFileName)) + + // the file was created after the blob, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // recreate the blob to have a later last modified time + scenarioHelper{}.generateBlobs(c, containerURL, blobList) + mockedRPC.reset() + + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + }) + } } // regular container->directory sync but destination is empty, so everything has to be transferred diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go index 264237257..7b38cbcda 100644 --- a/cmd/zt_sync_upload_test.go +++ b/cmd/zt_sync_upload_test.go @@ -33,46 +33,47 @@ import ( func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { bsu := getBSU() - // set up the source as a single file - srcDirName := scenarioHelper{}.generateLocalDirectory(c) - srcFileName := "singlefileisbest" - fileList := []string{srcFileName} - scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) - - // set up the destination container with a single blob - dstBlobName := srcFileName - containerURL, containerName := createNewContainer(c, bsu) - scenarioHelper{}.generateBlobs(c, containerURL, []string{dstBlobName}) - defer deleteContainer(c, containerURL) - c.Assert(containerURL, chk.NotNil) - - // set up interceptor - mockedRPC := interceptor{} - Rpc = mockedRPC.intercept - mockedRPC.init() - - // construct the raw input to simulate user input - rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dstBlobName) - raw := getDefaultRawInput(filepath.Join(srcDirName, srcFileName), rawBlobURLWithSAS.String()) - - // the blob was created after the file, so no sync should happen - runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.IsNil) - - // validate that the right number of transfers were scheduled - c.Assert(len(mockedRPC.transfers), chk.Equals, 0) - }) - - // recreate the file to have a later last modified time - scenarioHelper{}.generateFilesFromList(c, srcDirName, []string{srcFileName}) - mockedRPC.reset() - - // the file was created after the blob, so the sync should happen - runSyncAndVerify(c, raw, func(err error) { - c.Assert(err, chk.IsNil) - - validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) - }) + for _, srcFileName := range []string{"singlefileisbest", "打麻将.txt", "%4509%4254$85140&"} { + // set up the source as a single file + srcDirName := scenarioHelper{}.generateLocalDirectory(c) + fileList := []string{srcFileName} + scenarioHelper{}.generateFilesFromList(c, srcDirName, fileList) + + // set up the destination container with a single blob + dstBlobName := srcFileName + containerURL, containerName := createNewContainer(c, bsu) + scenarioHelper{}.generateBlobs(c, containerURL, []string{dstBlobName}) + defer deleteContainer(c, containerURL) + c.Assert(containerURL, chk.NotNil) + + // set up interceptor + mockedRPC := interceptor{} + Rpc = mockedRPC.intercept + mockedRPC.init() + + // construct the raw input to simulate user input + rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dstBlobName) + raw := getDefaultRawInput(filepath.Join(srcDirName, srcFileName), rawBlobURLWithSAS.String()) + + // the blob was created after the file, so no sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + // validate that the right number of transfers were scheduled + c.Assert(len(mockedRPC.transfers), chk.Equals, 0) + }) + + // recreate the file to have a later last modified time + scenarioHelper{}.generateFilesFromList(c, srcDirName, []string{srcFileName}) + mockedRPC.reset() + + // the file was created after the blob, so the sync should happen + runSyncAndVerify(c, raw, func(err error) { + c.Assert(err, chk.IsNil) + + validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + }) + } } // regular directory->container sync but destination is empty, so everything has to be transferred diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index c676b2642..7376aaa08 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -298,7 +298,16 @@ func fromToValue(from Location, to Location) FromTo { } func (l Location) IsRemote() bool { - return l == ELocation.BlobFS() || l == ELocation.Blob() || l == ELocation.File() + switch l { + case ELocation.BlobFS(), ELocation.Blob(), ELocation.File(): + return true + case ELocation.Local(), ELocation.Pipe(): + return false + default: + panic("unexpected location, please specify if it is remote") + } + + return false } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// From 46ce06804b25de4d46900610f024f9bb7ebece01 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 22 Feb 2019 00:50:14 -0800 Subject: [PATCH 51/64] Formalized output format --- cmd/cancel.go | 8 +- cmd/copy.go | 157 +++++++-------- cmd/copyDownloadBlobEnumerator.go | 2 +- cmd/copyDownloadBlobFSEnumerator.go | 2 +- cmd/env.go | 2 +- cmd/jobsList.go | 32 +++- cmd/jobsResume.go | 119 +++++++----- cmd/jobsShow.go | 103 +++++----- cmd/list.go | 35 ++-- cmd/make.go | 8 +- cmd/pause.go | 10 +- cmd/remove.go | 5 +- cmd/root.go | 24 ++- cmd/sync.go | 160 ++++++++-------- cmd/syncEnumerator.go | 12 +- common/fe-ste-models.go | 4 + common/lifecyleMgr.go | 284 ++++++++++++++++++---------- common/output.go | 81 ++++++++ common/rpc-models.go | 12 +- main.go | 6 +- ste/mgr-JobPartTransferMgr.go | 2 +- ste/xfer-URLToBlob.go | 6 +- ste/xfer-deleteBlob.go | 2 +- ste/xfer-deletefile.go | 2 +- 24 files changed, 632 insertions(+), 446 deletions(-) create mode 100644 common/output.go diff --git a/cmd/cancel.go b/cmd/cancel.go index 013ddbeaa..985fec741 100644 --- a/cmd/cancel.go +++ b/cmd/cancel.go @@ -28,6 +28,8 @@ import ( "github.com/spf13/cobra" ) +// TODO should this command be removed? Previously AzCopy was supposed to have an independent backend (out of proc) +// TODO but that's not the plan anymore type rawCancelCmdArgs struct { jobID string } @@ -81,15 +83,15 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("failed to parse user input due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to parse user input due to error " + err.Error()) } err = cooked.process() if err != nil { - glcm.Exit("failed to perform copy command due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to perform copy command due to error " + err.Error()) } - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, // hide features not relevant to BFS // TODO remove after preview release. diff --git a/cmd/copy.go b/cmd/copy.go index 627d4fc69..43ddf7c97 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -238,9 +238,7 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { // if redirection is triggered, avoid printing any output if cooked.isRedirection() { - cooked.output = common.EOutputFormat.None() - } else { - cooked.output.Parse(raw.output) + glcm.SetOutputFormat(common.EOutputFormat.None()) } // generate a unique job ID @@ -358,7 +356,6 @@ type cookedCopyCmdArgs struct { preserveLastModifiedTime bool md5ValidationOption common.HashValidationOption background bool - output common.OutputFormat acl string logVerbosity common.LogLevel cancelFromStdin bool @@ -408,7 +405,7 @@ func (cca *cookedCopyCmdArgs) process() error { } // if no error, the operation is now complete - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } return cca.processCopyJobPartOrders() } @@ -719,8 +716,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { // if blocking is specified to false, then another goroutine spawns and wait out the job func (cca *cookedCopyCmdArgs) waitUntilJobCompletion(blocking bool) { // print initial message to indicate that the job is starting - glcm.Info("\nJob " + cca.jobID.String() + " has started\n") - glcm.Info(fmt.Sprintf("Log file is located at: %s/%s.log", azcopyLogPathFolder, cca.jobID)) + glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s/%s.log", azcopyLogPathFolder, cca.jobID))) // initialize the times necessary to track progress cca.jobStartTime = time.Now() @@ -739,10 +735,10 @@ func (cca *cookedCopyCmdArgs) waitUntilJobCompletion(blocking bool) { func (cca *cookedCopyCmdArgs) Cancel(lcm common.LifecycleMgr) { // prompt for confirmation, except when: - // 1. output is in json format + // 1. output is not in text format // 2. azcopy was spawned by another process (cancelFromStdin indicates this) // 3. enumeration is complete - if !(cca.output == common.EOutputFormat.Json() || cca.cancelFromStdin || cca.isEnumerationComplete) { + if !(azcopyOutputFormat != common.EOutputFormat.Text() || cca.cancelFromStdin || cca.isEnumerationComplete) { answer := lcm.Prompt("The source enumeration is not complete, cancelling the job at this point means it cannot be resumed. Please confirm with y/n: ") // read a line from stdin, if the answer is not yes, then abort cancel by returning @@ -753,7 +749,7 @@ func (cca *cookedCopyCmdArgs) Cancel(lcm common.LifecycleMgr) { err := cookedCancelCmdArgs{jobID: cca.jobID}.process() if err != nil { - lcm.Exit("error occurred while cancelling the job "+cca.jobID.String()+". Failed with error "+err.Error(), common.EExitCode.Error()) + lcm.Error("error occurred while cancelling the job " + cca.jobID.String() + ": " + err.Error()) } } @@ -763,80 +759,77 @@ func (cca *cookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { Rpc(common.ERpcCmd.ListJobSummary(), &cca.jobID, &summary) jobDone := summary.JobStatus.IsJobDone() - // if json output is desired, simply marshal and return - // note that if job is already done, we simply exit - if cca.output == common.EOutputFormat.Json() { - //jsonOutput, err := json.MarshalIndent(summary, "", " ") - jsonOutput, err := json.Marshal(summary) - common.PanicIfErr(err) - - if jobDone { - exitCode := common.EExitCode.Success() - if summary.TransfersFailed > 0 { - exitCode = common.EExitCode.Error() - } - lcm.Exit(string(jsonOutput), exitCode) - } else { - lcm.Info(string(jsonOutput)) - return - } - } - // if json is not desired, and job is done, then we generate a special end message to conclude the job duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job + if jobDone { exitCode := common.EExitCode.Success() if summary.TransfersFailed > 0 { exitCode = common.EExitCode.Error() } - lcm.Exit(fmt.Sprintf( - "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n", - summary.JobID.String(), - ste.ToFixed(duration.Minutes(), 4), - summary.TotalTransfers, - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TransfersSkipped, - summary.TotalBytesTransferred, - summary.JobStatus), exitCode) - } - - // if json is not needed, and job is not done, then we generate a message that goes nicely on the same line - // display a scanning keyword if the job is not completely ordered - var scanningString = "" - if !summary.CompleteJobOrdered { - scanningString = " (scanning...)" - } - - // compute the average throughput for the last time interval - bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) - timeElapsed := time.Since(cca.intervalStartTime).Seconds() - throughPut := common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 - - // reset the interval timer and byte count - cca.intervalStartTime = time.Now() - cca.intervalBytesTransferred = summary.BytesOverWire - - // indicate whether constrained by disk or not - perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) - - // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. - if throughPut == 0 { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s %s", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, - summary.TotalTransfers, - scanningString, - perfString)) - } else { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s2-sec Throughput (Mb/s): %v%s", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, ste.ToFixed(throughPut, 4), diskString)) + + lcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + return fmt.Sprintf( + "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n", + summary.JobID.String(), + ste.ToFixed(duration.Minutes(), 4), + summary.TotalTransfers, + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TransfersSkipped, + summary.TotalBytesTransferred, + summary.JobStatus) + } + }, exitCode) } + + var computeThroughput = func() float64 { + // compute the average throughput for the last time interval + bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) + timeElapsed := time.Since(cca.intervalStartTime).Seconds() + + // reset the interval timer and byte count + cca.intervalStartTime = time.Now() + cca.intervalBytesTransferred = summary.BytesOverWire + + return common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 + } + + glcm.Progress(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + // if json is not needed, then we generate a message that goes nicely on the same line + // display a scanning keyword if the job is not completely ordered + var scanningString = " (scanning...)" + if summary.CompleteJobOrdered { + scanningString = "" + } + + throughput := computeThroughput() + throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) + if throughput == 0 { + // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. + throughputString = "" + } + + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + + return fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s%s%s", + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), + summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, throughputString, diskString) + } + }) } // Is disk speed looking like a constraint on throughput? Ignore the first little-while, @@ -909,17 +902,15 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("failed to parse user input due to error: "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to parse user input due to error: " + err.Error()) } - if cooked.output == common.EOutputFormat.Text() { - glcm.Info("Scanning...") - } + glcm.Info("Scanning...") cooked.commandString = copyHandlerUtil{}.ConstructCommandStringFromArgs() err = cooked.process() if err != nil { - glcm.Exit("failed to perform copy command due to error: "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to perform copy command due to error: " + err.Error()) } glcm.SurrenderControl() @@ -941,7 +932,6 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "exclude-blob-type", "", "optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from Container / Account. Use of "+ "this flag is not applicable for copying data from non azure-service to service. More than one blob should be separated by ';' ") // options change how the transfers are performed - cpCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json.") cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") cpCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "use this block(chunk) size when uploading/downloading to/from Azure Storage.") cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "None", "defines the type of blob at the destination. This is used in case of upload / account to account copy") @@ -967,8 +957,5 @@ func init() { // Hide the list-of-files flag since it is implemented only for Storage Explorer. cpCmd.PersistentFlags().MarkHidden("list-of-files") cpCmd.PersistentFlags().MarkHidden("include") - cpCmd.PersistentFlags().MarkHidden("output") - cpCmd.PersistentFlags().MarkHidden("stdin-enable") - cpCmd.PersistentFlags().MarkHidden("background-op") cpCmd.PersistentFlags().MarkHidden("cancel-from-stdin") } diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index 5824f245f..eb3f2857b 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -213,7 +213,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error { } // If there are no transfer to queue up, exit with message if len(e.Transfers) == 0 { - glcm.Exit(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination), 1) + glcm.Error(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination)) return nil } // dispatch the JobPart as Final Part of the Job diff --git a/cmd/copyDownloadBlobFSEnumerator.go b/cmd/copyDownloadBlobFSEnumerator.go index 8c4814147..cc9c20930 100644 --- a/cmd/copyDownloadBlobFSEnumerator.go +++ b/cmd/copyDownloadBlobFSEnumerator.go @@ -154,7 +154,7 @@ func (e *copyDownloadBlobFSEnumerator) enumerate(cca *cookedCopyCmdArgs) error { } // If there are no transfer to queue up, exit with message if len(e.Transfers) == 0 { - glcm.Exit(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination), 1) + glcm.Error(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination)) return nil } // dispatch the JobPart as Final Part of the Job diff --git a/cmd/env.go b/cmd/env.go index de985d2b0..032a7cda1 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -31,7 +31,7 @@ var envCmd = &cobra.Command{ env.Name, glcm.GetEnvironmentVariable(env), env.Description)) } - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, } diff --git a/cmd/jobsList.go b/cmd/jobsList.go index b4a8edfab..6f8fe0cc8 100644 --- a/cmd/jobsList.go +++ b/cmd/jobsList.go @@ -21,8 +21,10 @@ package cmd import ( + "encoding/json" "fmt" "sort" + "strings" "time" "github.com/Azure/azure-storage-azcopy/common" @@ -52,9 +54,9 @@ func init() { Run: func(cmd *cobra.Command, args []string) { err := HandleListJobsCommand() if err == nil { - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } else { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } }, } @@ -79,14 +81,24 @@ func PrintExistingJobIds(listJobResponse common.ListJobsResponse) error { // before displaying the jobs, sort them accordingly so that they are displayed in a consistent way sortJobs(listJobResponse.JobIDDetails) - glcm.Info("Existing Jobs ") - for index := 0; index < len(listJobResponse.JobIDDetails); index++ { - jobDetail := listJobResponse.JobIDDetails[index] - glcm.Info(fmt.Sprintf("JobId: %s\nStart Time: %s\nCommand: %s\n", - jobDetail.JobId.String(), - time.Unix(0, jobDetail.StartTime).Format(time.RFC850), - jobDetail.CommandString)) - } + glcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(listJobResponse) + common.PanicIfErr(err) + return string(jsonOutput) + } + + var sb strings.Builder + sb.WriteString("Existing Jobs \n") + for index := 0; index < len(listJobResponse.JobIDDetails); index++ { + jobDetail := listJobResponse.JobIDDetails[index] + sb.WriteString(fmt.Sprintf("JobId: %s\nStart Time: %s\nCommand: %s\n\n", + jobDetail.JobId.String(), + time.Unix(0, jobDetail.StartTime).Format(time.RFC850), + jobDetail.CommandString)) + } + return sb.String() + }, common.EExitCode.Success()) return nil } diff --git a/cmd/jobsResume.go b/cmd/jobsResume.go index 0e2d40d9a..0be05341b 100644 --- a/cmd/jobsResume.go +++ b/cmd/jobsResume.go @@ -22,6 +22,7 @@ package cmd import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -33,7 +34,8 @@ import ( ) // TODO the behavior of the resume command should be double-checked -// TODO ex: does it output json?? +// TODO figure out how to merge resume job with copy +// TODO the progress reporting code is almost the same as the copy command, the copy-paste should be avoided type resumeJobController struct { // generated jobID common.JobID @@ -54,8 +56,8 @@ type resumeJobController struct { // if blocking is specified to false, then another goroutine spawns and wait out the job func (cca *resumeJobController) waitUntilJobCompletion(blocking bool) { // print initial message to indicate that the job is starting - glcm.Info("\nJob " + cca.jobID.String() + " has started\n") - glcm.Info(fmt.Sprintf("Log file is located at: %s/%s.log", azcopyLogPathFolder, cca.jobID)) + glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s/%s.log", azcopyLogPathFolder, cca.jobID))) + // initialize the times necessary to track progress cca.jobStartTime = time.Now() cca.intervalStartTime = time.Now() @@ -74,7 +76,7 @@ func (cca *resumeJobController) waitUntilJobCompletion(blocking bool) { func (cca *resumeJobController) Cancel(lcm common.LifecycleMgr) { err := cookedCancelCmdArgs{jobID: cca.jobID}.process() if err != nil { - lcm.Exit("error occurred while cancelling the job "+cca.jobID.String()+". Failed with error "+err.Error(), common.EExitCode.Error()) + lcm.Error("error occurred while cancelling the job " + cca.jobID.String() + ". Failed with error " + err.Error()) } } @@ -86,58 +88,75 @@ func (cca *resumeJobController) ReportProgressOrExit(lcm common.LifecycleMgr) { // if json is not desired, and job is done, then we generate a special end message to conclude the job duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job + if jobDone { exitCode := common.EExitCode.Success() if summary.TransfersFailed > 0 { exitCode = common.EExitCode.Error() } - lcm.Exit(fmt.Sprintf( - "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nFinal Job Status: %v\n", - summary.JobID.String(), - ste.ToFixed(duration.Minutes(), 4), - summary.TotalTransfers, - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TransfersSkipped, - summary.JobStatus), exitCode) - } - // if json is not needed, and job is not done, then we generate a message that goes nicely on the same line - // display a scanning keyword if the job is not completely ordered - var scanningString = "" - if !summary.CompleteJobOrdered { - scanningString = "(scanning...)" + lcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + return fmt.Sprintf( + "\n\nJob %s summary\nElapsed Time (Minutes): %v\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nTotalBytesTransferred: %v\nFinal Job Status: %v\n", + summary.JobID.String(), + ste.ToFixed(duration.Minutes(), 4), + summary.TotalTransfers, + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TransfersSkipped, + summary.TotalBytesTransferred, + summary.JobStatus) + } + }, exitCode) } - // compute the average throughput for the last time interval - bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) - timeElapsed := time.Since(cca.intervalStartTime).Seconds() - throughPut := common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 + var computeThroughput = func() float64 { + // compute the average throughput for the last time interval + bytesInMb := float64(float64(summary.BytesOverWire-cca.intervalBytesTransferred) / float64(1024*1024)) + timeElapsed := time.Since(cca.intervalStartTime).Seconds() - // reset the interval timer and byte count - cca.intervalStartTime = time.Now() - cca.intervalBytesTransferred = summary.BytesOverWire - - // indicate whether constrained by disk or not - perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) - - // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. - if throughPut == 0 { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s %s", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, - summary.TotalTransfers, - scanningString, - perfString)) - } else { - glcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped %v Total %s, %s2-sec Throughput (Mb/s): %v%s", - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), - summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, ste.ToFixed(throughPut, 4), diskString)) + // reset the interval timer and byte count + cca.intervalStartTime = time.Now() + cca.intervalBytesTransferred = summary.BytesOverWire + + return common.Iffloat64(timeElapsed != 0, bytesInMb/timeElapsed, 0) * 8 } + + glcm.Progress(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) + } else { + // if json is not needed, then we generate a message that goes nicely on the same line + // display a scanning keyword if the job is not completely ordered + var scanningString = " (scanning...)" + if summary.CompleteJobOrdered { + scanningString = "" + } + + throughput := computeThroughput() + throughputString := fmt.Sprintf("2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) + if throughput == 0 { + // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it. + throughputString = "" + } + + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + + return fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s%s%s", + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed+summary.TransfersSkipped), + summary.TransfersSkipped, summary.TotalTransfers, scanningString, perfString, throughputString, diskString) + } + }) } func init() { @@ -163,9 +182,9 @@ func init() { Run: func(cmd *cobra.Command, args []string) { err := resumeCmdArgs.process() if err != nil { - glcm.Exit(fmt.Sprintf("failed to perform resume command due to error: %s", err.Error()), common.EExitCode.Error()) + glcm.Error(fmt.Sprintf("failed to perform resume command due to error: %s", err.Error())) } - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, } @@ -236,7 +255,7 @@ func (rca resumeCmdArgs) process() error { &common.GetJobFromToRequest{JobID: jobID}, &getJobFromToResponse) if getJobFromToResponse.ErrorMsg != "" { - glcm.Exit(getJobFromToResponse.ErrorMsg, common.EExitCode.Error()) + glcm.Error(getJobFromToResponse.ErrorMsg) } ctx := context.TODO() @@ -279,7 +298,7 @@ func (rca resumeCmdArgs) process() error { &resumeJobResponse) if !resumeJobResponse.CancelledPauseResumed { - glcm.Exit(resumeJobResponse.ErrorMsg, common.EExitCode.Error()) + glcm.Error(resumeJobResponse.ErrorMsg) } controller := resumeJobController{jobID: jobID} diff --git a/cmd/jobsShow.go b/cmd/jobsShow.go index e7b4e7778..6fbd5ee7e 100644 --- a/cmd/jobsShow.go +++ b/cmd/jobsShow.go @@ -23,6 +23,7 @@ package cmd import ( "errors" "fmt" + "strings" "encoding/json" @@ -33,7 +34,6 @@ import ( type ListReq struct { JobID common.JobID OfStatus string - Output string } func init() { @@ -63,15 +63,12 @@ func init() { listRequest := common.ListRequest{} listRequest.JobID = commandLineInput.JobID listRequest.OfStatus = commandLineInput.OfStatus - err := listRequest.Output.Parse(commandLineInput.Output) - if err != nil { - glcm.Exit(fmt.Errorf("error parsing the given output format %s", commandLineInput.Output).Error(), common.EExitCode.Error()) - } - err = HandleShowCommand(listRequest) + + err := HandleShowCommand(listRequest) if err == nil { - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } else { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } }, } @@ -80,8 +77,6 @@ func init() { // filters shJob.PersistentFlags().StringVar(&commandLineInput.OfStatus, "with-status", "", "only list the transfers of job with this status, available values: Started, Success, Failed") - // filters - shJob.PersistentFlags().StringVar(&commandLineInput.Output, "output", "text", "format of the command's output, the choices include: text, json") } // handles the list command @@ -92,7 +87,7 @@ func HandleShowCommand(listRequest common.ListRequest) error { resp := common.ListJobSummaryResponse{} rpcCmd = common.ERpcCmd.ListJobSummary() Rpc(rpcCmd, &listRequest.JobID, &resp) - PrintJobProgressSummary(listRequest.Output, resp) + PrintJobProgressSummary(resp) } else { lsRequest := common.ListJobTransfersRequest{} lsRequest.JobID = listRequest.JobID @@ -105,67 +100,59 @@ func HandleShowCommand(listRequest common.ListRequest) error { resp := common.ListJobTransfersResponse{} rpcCmd = common.ERpcCmd.ListJobTransfers() Rpc(rpcCmd, lsRequest, &resp) - PrintJobTransfers(listRequest.Output, resp) + PrintJobTransfers(resp) } return nil } // PrintJobTransfers prints the response of listOrder command when list Order command requested the list of specific transfer of an existing job -func PrintJobTransfers(outputForamt common.OutputFormat, listTransfersResponse common.ListJobTransfersResponse) { - if outputForamt == common.EOutputFormat.Json() { - var exitCode = common.EExitCode.Success() - if listTransfersResponse.ErrorMsg != "" { - exitCode = common.EExitCode.Error() - } - //jsonOutput, err := json.MarshalIndent(listTransfersResponse, "", " ") - jsonOutput, err := json.Marshal(listTransfersResponse) - common.PanicIfErr(err) - glcm.Exit(string(jsonOutput), exitCode) - return - } +func PrintJobTransfers(listTransfersResponse common.ListJobTransfersResponse) { if listTransfersResponse.ErrorMsg != "" { - glcm.Exit("request failed with following message "+listTransfersResponse.ErrorMsg, common.EExitCode.Error()) - return + glcm.Error("request failed with following message " + listTransfersResponse.ErrorMsg) } - glcm.Info("----------- Transfers for JobId " + listTransfersResponse.JobID.String() + " -----------") - for index := 0; index < len(listTransfersResponse.Details); index++ { - glcm.Info("transfer--> source: " + listTransfersResponse.Details[index].Src + " destination: " + - listTransfersResponse.Details[index].Dst + " status " + listTransfersResponse.Details[index].TransferStatus.String()) - } + glcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(listTransfersResponse) + common.PanicIfErr(err) + return string(jsonOutput) + } + + var sb strings.Builder + sb.WriteString("----------- Transfers for JobId " + listTransfersResponse.JobID.String() + " -----------\n") + for index := 0; index < len(listTransfersResponse.Details); index++ { + sb.WriteString("transfer--> source: " + listTransfersResponse.Details[index].Src + " destination: " + + listTransfersResponse.Details[index].Dst + " status " + listTransfersResponse.Details[index].TransferStatus.String() + "\n") + } + + return sb.String() + }, common.EExitCode.Success()) } // PrintJobProgressSummary prints the response of listOrder command when listOrder command requested the progress summary of an existing job -func PrintJobProgressSummary(outputFormat common.OutputFormat, summary common.ListJobSummaryResponse) { +func PrintJobProgressSummary(summary common.ListJobSummaryResponse) { + if summary.ErrorMsg != "" { + glcm.Error("list progress summary of job failed because " + summary.ErrorMsg) + } + // Reset the bytes over the wire counter summary.BytesOverWire = 0 - // If the output format is Json, check the summary's error Message. - // If there is an error message, then the exit code is error - // else the exit code is success. - // Marshal the summary and print in the Json format. - if outputFormat == common.EOutputFormat.Json() { - var exitCode = common.EExitCode.Success() - if summary.ErrorMsg != "" { - exitCode = common.EExitCode.Error() + glcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) } - //jsonOutput, err := json.MarshalIndent(summary, "", " ") - jsonOutput, err := json.Marshal(summary) - common.PanicIfErr(err) - glcm.Exit(string(jsonOutput), exitCode) - return - } - if summary.ErrorMsg != "" { - glcm.Exit("list progress summary of job failed because "+summary.ErrorMsg, common.EExitCode.Error()) - } - glcm.Info(fmt.Sprintf( - "\nJob %s summary\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nFinal Job Status: %v\n", - summary.JobID.String(), - summary.TotalTransfers, - summary.TransfersCompleted, - summary.TransfersFailed, - summary.TransfersSkipped, - summary.JobStatus, - )) + return fmt.Sprintf( + "\nJob %s summary\nTotal Number Of Transfers: %v\nNumber of Transfers Completed: %v\nNumber of Transfers Failed: %v\nNumber of Transfers Skipped: %v\nFinal Job Status: %v\n", + summary.JobID.String(), + summary.TotalTransfers, + summary.TransfersCompleted, + summary.TransfersFailed, + summary.TransfersSkipped, + summary.JobStatus, + ) + }, common.EExitCode.Success()) } diff --git a/cmd/list.go b/cmd/list.go index 375b1b26a..c9cec3dc4 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -22,7 +22,6 @@ package cmd import ( "context" - "encoding/json" "errors" "fmt" "net/url" @@ -36,7 +35,6 @@ import ( func init() { var sourcePath = "" - var outputRaw = "" // listContainerCmd represents the list container command // listContainer list the blobs inside the container or virtual directory inside the container listContainerCmd := &cobra.Command{ @@ -61,26 +59,23 @@ func init() { // verifying the location type location := inferArgumentLocation(sourcePath) if location != location.Blob() { - glcm.Exit("invalid path passed for listing. given source is of type "+location.String()+" while expect is container / container path ", common.EExitCode.Error()) + glcm.Error("invalid path passed for listing. given source is of type " + location.String() + " while expect is container / container path ") } - var output common.OutputFormat - output.Parse(outputRaw) - err := HandleListContainerCommand(sourcePath, output) + err := HandleListContainerCommand(sourcePath) if err == nil { - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } else { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } }, } rootCmd.AddCommand(listContainerCmd) - listContainerCmd.PersistentFlags().StringVar(&outputRaw, "output", "text", "format of the command's output, the choices include: text, json") } // HandleListContainerCommand handles the list container command -func HandleListContainerCommand(source string, outputFormat common.OutputFormat) (err error) { +func HandleListContainerCommand(source string) (err error) { // TODO: Temporarily use context.TODO(), this should be replaced with a root context from main. ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) @@ -152,27 +147,19 @@ func HandleListContainerCommand(source string, outputFormat common.OutputFormat) summary.Blobs = append(summary.Blobs, blobName) } marker = listBlob.NextMarker - printListContainerResponse(&summary, outputFormat) + printListContainerResponse(&summary) } return nil } // printListContainerResponse prints the list container response -func printListContainerResponse(lsResponse *common.ListContainerResponse, outputFormat common.OutputFormat) { +func printListContainerResponse(lsResponse *common.ListContainerResponse) { if len(lsResponse.Blobs) == 0 { return } - if outputFormat == common.EOutputFormat.Json() { - //marshaledData, err := json.MarshalIndent(lsResponse, "", " ") - marshaledData, err := json.Marshal(lsResponse) - if err != nil { - panic(fmt.Errorf("error listing the source. Failed with error %s", err)) - } - glcm.Info(string(marshaledData)) - } else { - for index := 0; index < len(lsResponse.Blobs); index++ { - glcm.Info(lsResponse.Blobs[index]) - } + // TODO determine what's the best way to display the blobs in JSON + // TODO no partner team needs this functionality right now so the blobs are just outputted as info + for index := 0; index < len(lsResponse.Blobs); index++ { + glcm.Info(lsResponse.Blobs[index]) } - lsResponse.Blobs = nil } diff --git a/cmd/make.go b/cmd/make.go index 9cf5e864d..642591596 100644 --- a/cmd/make.go +++ b/cmd/make.go @@ -200,15 +200,17 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cookedArgs, err := rawArgs.cook() if err != nil { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } err = cookedArgs.process() if err != nil { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } - glcm.Exit("Successfully created the resource.", common.EExitCode.Success()) + glcm.Exit(func(format common.OutputFormat) string { + return "Successfully created the resource." + }, common.EExitCode.Success()) }, } diff --git a/cmd/pause.go b/cmd/pause.go index 3ea6fca3a..470c0ef9e 100644 --- a/cmd/pause.go +++ b/cmd/pause.go @@ -27,6 +27,8 @@ import ( "github.com/spf13/cobra" ) +// TODO should this command be removed? Previously AzCopy was supposed to have an independent backend (out of proc) +// TODO but that's not the plan anymore func init() { var commandLineInput = "" @@ -49,7 +51,7 @@ func init() { }, Run: func(cmd *cobra.Command, args []string) { HandlePauseCommand(commandLineInput) - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) }, // hide features not relevant to BFS // TODO remove after preview release @@ -66,10 +68,12 @@ func HandlePauseCommand(jobIdString string) { jobID, err := common.ParseJobID(jobIdString) if err != nil { // If parsing gives an error, hence it is not a valid JobId format - glcm.Exit("invalid jobId string passed. Failed while parsing string to jobId", common.EExitCode.Error()) + glcm.Error("invalid jobId string passed. Failed while parsing string to jobId") } var pauseJobResponse common.CancelPauseResumeResponse Rpc(common.ERpcCmd.PauseJob(), jobID, &pauseJobResponse) - glcm.Exit("Job "+jobID.String()+" paused successfully", common.EExitCode.Success()) + glcm.Exit(func(format common.OutputFormat) string { + return "Job " + jobID.String() + " paused successfully" + }, common.EExitCode.Success()) } diff --git a/cmd/remove.go b/cmd/remove.go index 53278e6ab..cbefbf417 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -60,12 +60,12 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("failed to parse user input due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to parse user input due to error " + err.Error()) } cooked.commandString = copyHandlerUtil{}.ConstructCommandStringFromArgs() err = cooked.process() if err != nil { - glcm.Exit("failed to perform copy command due to error "+err.Error(), common.EExitCode.Error()) + glcm.Error("failed to perform copy command due to error " + err.Error()) } glcm.SurrenderControl() @@ -75,5 +75,4 @@ func init() { deleteCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Filter: Look into sub-directories recursively when deleting from container.") deleteCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "WARNING", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") - deleteCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") } diff --git a/cmd/root.go b/cmd/root.go index a15970e9b..65326c601 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,6 +34,8 @@ import ( var azcopyAppPathFolder string var azcopyLogPathFolder string +var outputFormatRaw string +var azcopyOutputFormat common.OutputFormat // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ @@ -41,13 +43,20 @@ var rootCmd = &cobra.Command{ Use: "azcopy", Short: rootCmdShortDescription, Long: rootCmdLongDescription, - PersistentPreRun: func(cmd *cobra.Command, args []string) { + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + err := azcopyOutputFormat.Parse(outputFormatRaw) + glcm.SetOutputFormat(azcopyOutputFormat) + if err != nil { + return err + } + // spawn a routine to fetch and compare the local application's version against the latest version available // if there's a newer version that can be used, then write the suggestion to stderr // however if this takes too long the message won't get printed // Note: this function is only triggered for non-help commands go detectNewVersion() + return nil }, } @@ -61,16 +70,20 @@ func Execute(azsAppPathFolder, logPathFolder string) { azcopyLogPathFolder = logPathFolder if err := rootCmd.Execute(); err != nil { - glcm.Exit(err.Error(), common.EExitCode.Error()) + glcm.Error(err.Error()) } else { // our commands all control their own life explicitly with the lifecycle manager // only help commands reach this point // execute synchronously before exiting detectNewVersion() - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } } +func init() { + rootCmd.PersistentFlags().StringVar(&outputFormatRaw, "output", "text", "format of the command's output, the choices include: text, json.") +} + func detectNewVersion() { const versionMetadataUrl = "https://aka.ms/azcopyv10-version-metadata" @@ -132,7 +145,8 @@ func detectNewVersion() { if v1.OlderThan(*v2) { executablePathSegments := strings.Split(strings.Replace(os.Args[0], "\\", "/", -1), "/") executableName := executablePathSegments[len(executablePathSegments)-1] - // print to stderr instead of stdout, in case the output is in other formats - glcm.Error(executableName + ": A newer version " + remoteVersion + " is available to download\n") + + // output in info mode instead of stderr, as it was crashing CI jobs of some people + glcm.Info(executableName + ": A newer version " + remoteVersion + " is available to download\n") } } diff --git a/cmd/sync.go b/cmd/sync.go index 1e2d80c55..8fedf387d 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -51,7 +51,6 @@ type rawSyncCmdArgs struct { include string exclude string followSymlinks bool - output string md5ValidationOption string // this flag indicates the user agreement with respect to deleting the extra files at the destination // which do not exists at source. With this flag turned on/off, users will not be asked for permission. @@ -149,11 +148,6 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) { return cooked, err } - err = cooked.output.Parse(raw.output) - if err != nil { - return cooked, err - } - return cooked, nil } @@ -175,7 +169,6 @@ type cookedSyncCmdArgs struct { md5ValidationOption common.HashValidationOption blockSize uint32 logVerbosity common.LogLevel - output common.OutputFormat // commandString hold the user given command which is logged to the Job log file commandString string @@ -251,8 +244,7 @@ func (cca *cookedSyncCmdArgs) scanningComplete() bool { // if blocking is specified to false, then another goroutine spawns and wait out the job func (cca *cookedSyncCmdArgs) waitUntilJobCompletion(blocking bool) { // print initial message to indicate that the job is starting - glcm.Info("\nJob " + cca.jobID.String() + " has started\n") - glcm.Info(fmt.Sprintf("Log file is located at: %s/%s.log", azcopyLogPathFolder, cca.jobID)) + glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s/%s.log", azcopyLogPathFolder, cca.jobID))) // initialize the times necessary to track progress cca.jobStartTime = time.Now() @@ -270,10 +262,8 @@ func (cca *cookedSyncCmdArgs) waitUntilJobCompletion(blocking bool) { } func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { - // prompt for confirmation, except when: - // 1. output is in json format - // 2. enumeration is complete - if cca.output == common.EOutputFormat.Text() && !cca.isEnumerationComplete { + // prompt for confirmation, except when enumeration is complete + if !cca.isEnumerationComplete { answer := lcm.Prompt("The source enumeration is not complete, cancelling the job at this point means it cannot be resumed. Please confirm with y/n: ") // read a line from stdin, if the answer is not yes, then abort cancel by returning @@ -284,16 +274,53 @@ func (cca *cookedSyncCmdArgs) Cancel(lcm common.LifecycleMgr) { err := cookedCancelCmdArgs{jobID: cca.jobID}.process() if err != nil { - lcm.Exit("error occurred while cancelling the job "+cca.jobID.String()+". Failed with error "+err.Error(), common.EExitCode.Error()) + lcm.Error("error occurred while cancelling the job " + cca.jobID.String() + ". Failed with error " + err.Error()) } } -func (cca *cookedSyncCmdArgs) reportScanningProgress(lcm common.LifecycleMgr, throughputString string) { - lcm.Progress(fmt.Sprintf("%v Files Scanned at Source, %v Files Scanned at Destination%s", - atomic.LoadUint64(&cca.atomicSourceFilesScanned), atomic.LoadUint64(&cca.atomicDestinationFilesScanned), throughputString)) +type scanningProgressJsonTemplate struct { + FilesScannedAtSource uint64 + FilesScannedAtDestination uint64 +} + +func (cca *cookedSyncCmdArgs) reportScanningProgress(lcm common.LifecycleMgr, throughput float64) { + + lcm.Progress(func(format common.OutputFormat) string { + srcScanned := atomic.LoadUint64(&cca.atomicSourceFilesScanned) + dstScanned := atomic.LoadUint64(&cca.atomicDestinationFilesScanned) + + if format == common.EOutputFormat.Json() { + jsonOutputTemplate := scanningProgressJsonTemplate{ + FilesScannedAtSource: srcScanned, + FilesScannedAtDestination: dstScanned, + } + outputString, err := json.Marshal(jsonOutputTemplate) + common.PanicIfErr(err) + return string(outputString) + } + + // text output + throughputString := "" + if cca.firstPartOrdered() { + throughputString = fmt.Sprintf(", 2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) + } + return fmt.Sprintf("%v Files Scanned at Source, %v Files Scanned at Destination%s", + srcScanned, dstScanned, throughputString) + }) +} + +func (cca *cookedSyncCmdArgs) getJsonOfSyncJobSummary(summary common.ListSyncJobSummaryResponse) string { + // TODO figure out if deletions should be done by the enumeration engine or not + // TODO if not, remove this so that we get the proper number from the ste + summary.DeleteTotalTransfers = cca.getDeletionCount() + summary.DeleteTransfersCompleted = cca.getDeletionCount() + jsonOutput, err := json.Marshal(summary) + common.PanicIfErr(err) + return string(jsonOutput) } func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { + duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job var summary common.ListSyncJobSummaryResponse var throughput float64 var jobDone bool @@ -315,54 +342,24 @@ func (cca *cookedSyncCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) { // first part not dispatched, and we are still scanning // so a special message is outputted to notice the user that we are not stalling - if !jobDone && !cca.scanningComplete() { - // skip the interactive message if we were triggered by another tool - if cca.output == common.EOutputFormat.Json() { - return - } - - var throughputString string - if cca.firstPartOrdered() { - throughputString = fmt.Sprintf(", 2-sec Throughput (Mb/s): %v", ste.ToFixed(throughput, 4)) - } - - cca.reportScanningProgress(lcm, throughputString) + if !cca.scanningComplete() { + cca.reportScanningProgress(lcm, throughput) return } - // if json output is desired, simply marshal and return - // note that if job is already done, we simply exit - if cca.output == common.EOutputFormat.Json() { - // TODO figure out if deletions should be done by the enumeration engine or not - // TODO if not, remove this so that we get the proper number from the ste - summary.DeleteTotalTransfers = cca.getDeletionCount() - summary.DeleteTransfersCompleted = cca.getDeletionCount() - - //jsonOutput, err := json.MarshalIndent(summary, "", " ") - jsonOutput, err := json.Marshal(summary) - common.PanicIfErr(err) - - if jobDone { - exitCode := common.EExitCode.Success() - if summary.CopyTransfersFailed+summary.DeleteTransfersFailed > 0 { - exitCode = common.EExitCode.Error() - } - lcm.Exit(string(jsonOutput), exitCode) - } else { - lcm.Info(string(jsonOutput)) - return - } - } - - // if json is not desired, and job is done, then we generate a special end message to conclude the job - duration := time.Now().Sub(cca.jobStartTime) // report the total run time of the job if jobDone { exitCode := common.EExitCode.Success() if summary.CopyTransfersFailed+summary.DeleteTransfersFailed > 0 { exitCode = common.EExitCode.Error() } - lcm.Exit(fmt.Sprintf( - ` + + lcm.Exit(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + return cca.getJsonOfSyncJobSummary(summary) + } + + return fmt.Sprintf( + ` Job %s Summary Files Scanned at Source: %v Files Scanned at Destination: %v @@ -375,27 +372,35 @@ Total Number of Bytes Transferred: %v Total Number of Bytes Enumerated: %v Final Job Status: %v `, - summary.JobID.String(), - atomic.LoadUint64(&cca.atomicSourceFilesScanned), - atomic.LoadUint64(&cca.atomicDestinationFilesScanned), - ste.ToFixed(duration.Minutes(), 4), - summary.CopyTotalTransfers, - summary.CopyTransfersCompleted, - summary.CopyTransfersFailed, - cca.atomicDeletionCount, - summary.TotalBytesTransferred, - summary.TotalBytesEnumerated, - summary.JobStatus), exitCode) + summary.JobID.String(), + atomic.LoadUint64(&cca.atomicSourceFilesScanned), + atomic.LoadUint64(&cca.atomicDestinationFilesScanned), + ste.ToFixed(duration.Minutes(), 4), + summary.CopyTotalTransfers, + summary.CopyTransfersCompleted, + summary.CopyTransfersFailed, + cca.atomicDeletionCount, + summary.TotalBytesTransferred, + summary.TotalBytesEnumerated, + summary.JobStatus) + + }, exitCode) } - // indicate whether constrained by disk or not - perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) + lcm.Progress(func(format common.OutputFormat) string { + if format == common.EOutputFormat.Json() { + return cca.getJsonOfSyncJobSummary(summary) + } + + // indicate whether constrained by disk or not + perfString, diskString := getPerfDisplayText(summary.PerfStrings, summary.IsDiskConstrained, duration) - lcm.Progress(fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total%s, 2-sec Throughput (Mb/s): %v%s", - summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted, - summary.CopyTransfersFailed+summary.DeleteTransfersFailed, - summary.CopyTotalTransfers+summary.DeleteTotalTransfers-(summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted+summary.CopyTransfersFailed+summary.DeleteTransfersFailed), - summary.CopyTotalTransfers+summary.DeleteTotalTransfers, perfString, ste.ToFixed(throughput, 4), diskString)) + return fmt.Sprintf("%v Done, %v Failed, %v Pending, %v Total%s, 2-sec Throughput (Mb/s): %v%s", + summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted, + summary.CopyTransfersFailed+summary.DeleteTransfersFailed, + summary.CopyTotalTransfers+summary.DeleteTotalTransfers-(summary.CopyTransfersCompleted+summary.DeleteTransfersCompleted+summary.CopyTransfersFailed+summary.DeleteTransfersFailed), + summary.CopyTotalTransfers+summary.DeleteTotalTransfers, perfString, ste.ToFixed(throughput, 4), diskString) + }) } func (cca *cookedSyncCmdArgs) process() (err error) { @@ -479,12 +484,12 @@ func init() { Run: func(cmd *cobra.Command, args []string) { cooked, err := raw.cook() if err != nil { - glcm.Exit("error parsing the input given by the user. Failed with error "+err.Error(), common.EExitCode.Error()) + glcm.Error("error parsing the input given by the user. Failed with error " + err.Error()) } cooked.commandString = copyHandlerUtil{}.ConstructCommandStringFromArgs() err = cooked.process() if err != nil { - glcm.Exit("Cannot perform sync due to error: "+err.Error(), common.EExitCode.Error()) + glcm.Error("Cannot perform sync due to error: " + err.Error()) } glcm.SurrenderControl() @@ -496,7 +501,6 @@ func init() { syncCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 0, "use this block(chunk) size when uploading/downloading to/from Azure Storage.") syncCmd.PersistentFlags().StringVar(&raw.include, "include", "", "only include files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") - syncCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json") syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "false", "defines whether to delete extra files from the destination that are not present at the source. Could be set to true, false, or prompt. "+ "If set to prompt, user will be asked a question before scheduling files/blobs for deletion.") diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index abb649042..03170c522 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -150,11 +150,15 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator func quitIfInSync(transferJobInitiated, anyDestinationFileDeleted bool, cca *cookedSyncCmdArgs) { if !transferJobInitiated && !anyDestinationFileDeleted { - cca.reportScanningProgress(glcm, "") - glcm.Exit("The source and destination are already in sync.", common.EExitCode.Success()) + cca.reportScanningProgress(glcm, 0) + glcm.Exit(func(format common.OutputFormat) string { + return "The source and destination are already in sync." + }, common.EExitCode.Success()) } else if !transferJobInitiated && anyDestinationFileDeleted { // some files were deleted but no transfer scheduled - cca.reportScanningProgress(glcm, "") - glcm.Exit("The source and destination are now in sync.", common.EExitCode.Success()) + cca.reportScanningProgress(glcm, 0) + glcm.Exit(func(format common.OutputFormat) string { + return "The source and destination are now in sync." + }, common.EExitCode.Success()) } } diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 7376aaa08..32e4b81ba 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -134,6 +134,10 @@ func (of *OutputFormat) Parse(s string) error { return err } +func (of OutputFormat) String() string { + return enum.StringInt(of, reflect.TypeOf(of)) +} + var EExitCode = ExitCode(0) type ExitCode uint32 diff --git a/common/lifecyleMgr.go b/common/lifecyleMgr.go index 3bb200336..fdf4f4242 100644 --- a/common/lifecyleMgr.go +++ b/common/lifecyleMgr.go @@ -19,6 +19,7 @@ var lcm = func() (lcmgr *lifecycleMgr) { msgQueue: make(chan outputMessage, 1000), progressCache: "", cancelChannel: make(chan os.Signal, 1), + outputFormat: EOutputFormat.Text(), // output text by default } // kick off the single routine that processes output @@ -33,45 +34,34 @@ var lcm = func() (lcmgr *lifecycleMgr) { // create a public interface so that consumers outside of this package can refer to the lifecycle manager // but they would not be able to instantiate one type LifecycleMgr interface { - Progress(string) // print on the same line over and over again, not allowed to float up + Init(outputBuilder) // let the user know the job has started and initial information like log location + Progress(outputBuilder) // print on the same line over and over again, not allowed to float up + Exit(outputBuilder, ExitCode) // indicates successful execution exit after printing, allow user to specify exit code Info(string) // simple print, allowed to float up + Error(string) // indicates fatal error, exit after printing, exit code is always Failed (1) Prompt(string) string // ask the user a question(after erasing the progress), then return the response - Exit(string, ExitCode) // exit after printing - Error(string) // print to stderr + StdError(string) // print to stderr SurrenderControl() // give up control, this should never return InitiateProgressReporting(WorkController, bool) // start writing progress with another routine GetEnvironmentVariable(EnvironmentVariable) string // get the environment variable or its default value + SetOutputFormat(OutputFormat) // change the output format of the entire application } func GetLifecycleMgr() LifecycleMgr { return lcm } -var eMessageType = outputMessageType(0) - -// outputMessageType defines the nature of the output, ex: progress report, job summary, or error -type outputMessageType uint8 - -func (outputMessageType) Progress() outputMessageType { return outputMessageType(0) } // should be printed on the same line over and over again, not allowed to float up -func (outputMessageType) Info() outputMessageType { return outputMessageType(1) } // simple print, allowed to float up -func (outputMessageType) Exit() outputMessageType { return outputMessageType(2) } // exit after printing -func (outputMessageType) Prompt() outputMessageType { return outputMessageType(3) } // ask the user a question after erasing the progress -func (outputMessageType) Error() outputMessageType { return outputMessageType(4) } // print to stderr - -// defines the output and how it should be handled -type outputMessage struct { - msgContent string - msgType outputMessageType - exitCode ExitCode // only for when the application is meant to exit after printing (i.e. Error or Final) - inputChannel chan<- string // support getting a response from the user -} - // single point of control for all outputs type lifecycleMgr struct { msgQueue chan outputMessage progressCache string // useful for keeping job progress on the last line cancelChannel chan os.Signal waitEverCalled int32 + outputFormat OutputFormat +} + +func (lcm *lifecycleMgr) SetOutputFormat(format OutputFormat) { + lcm.outputFormat = format } func (lcm *lifecycleMgr) checkAndStartCPUProfiling() { @@ -84,10 +74,10 @@ func (lcm *lifecycleMgr) checkAndStartCPUProfiling() { lcm.Info(fmt.Sprintf("pprof start CPU profiling, and saving profiling data to: %q", cpuProfilePath)) f, err := os.Create(cpuProfilePath) if err != nil { - lcm.Exit(fmt.Sprintf("Fail to create file for CPU profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to create file for CPU profiling, %v", err)) } if err := pprof.StartCPUProfile(f); err != nil { - lcm.Exit(fmt.Sprintf("Fail to start CPU profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to start CPU profiling, %v", err)) } } } @@ -107,11 +97,11 @@ func (lcm *lifecycleMgr) checkAndTriggerMemoryProfiling() { lcm.Info(fmt.Sprintf("pprof start memory profiling, and saving profiling data to: %q", memProfilePath)) f, err := os.Create(memProfilePath) if err != nil { - lcm.Exit(fmt.Sprintf("Fail to create file for memory profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to create file for memory profiling, %v", err)) } runtime.GC() if err := pprof.WriteHeapProfile(f); err != nil { - lcm.Exit(fmt.Sprintf("Fail to start memory profiling, %v", err), EExitCode.Error()) + lcm.Error(fmt.Sprintf("Fail to start memory profiling, %v", err)) } if err := f.Close(); err != nil { lcm.Info(fmt.Sprintf("Fail to close memory profiling file, %v", err)) @@ -119,24 +109,36 @@ func (lcm *lifecycleMgr) checkAndTriggerMemoryProfiling() { } } -func (lcm *lifecycleMgr) Progress(msg string) { +func (lcm *lifecycleMgr) Init(o outputBuilder) { lcm.msgQueue <- outputMessage{ - msgContent: msg, - msgType: eMessageType.Progress(), + msgContent: o(lcm.outputFormat), + msgType: eOutputMessageType.Init(), + } +} + +func (lcm *lifecycleMgr) Progress(o outputBuilder) { + messageContent := "" + if o != nil { + messageContent = o(lcm.outputFormat) + } + + lcm.msgQueue <- outputMessage{ + msgContent: messageContent, + msgType: eOutputMessageType.Progress(), } } func (lcm *lifecycleMgr) Info(msg string) { lcm.msgQueue <- outputMessage{ msgContent: msg, - msgType: eMessageType.Info(), + msgType: eOutputMessageType.Info(), } } -func (lcm *lifecycleMgr) Error(msg string) { +func (lcm *lifecycleMgr) StdError(msg string) { lcm.msgQueue <- outputMessage{ msgContent: msg, - msgType: eMessageType.Error(), + msgType: eOutputMessageType.StdError(), } } @@ -144,7 +146,7 @@ func (lcm *lifecycleMgr) Prompt(msg string) string { expectedInputChannel := make(chan string, 1) lcm.msgQueue <- outputMessage{ msgContent: msg, - msgType: eMessageType.Prompt(), + msgType: eOutputMessageType.Prompt(), inputChannel: expectedInputChannel, } @@ -152,7 +154,7 @@ func (lcm *lifecycleMgr) Prompt(msg string) string { return <-expectedInputChannel } -func (lcm *lifecycleMgr) Exit(msg string, exitCode ExitCode) { +func (lcm *lifecycleMgr) Error(msg string) { // Check if need to do memory profiling, and do memory profiling accordingly before azcopy exits. lcm.checkAndTriggerMemoryProfiling() @@ -161,7 +163,29 @@ func (lcm *lifecycleMgr) Exit(msg string, exitCode ExitCode) { lcm.msgQueue <- outputMessage{ msgContent: msg, - msgType: eMessageType.Exit(), + msgType: eOutputMessageType.Error(), + exitCode: EExitCode.Error(), + } + + // stall forever until the success message is printed and program exits + lcm.SurrenderControl() +} + +func (lcm *lifecycleMgr) Exit(o outputBuilder, exitCode ExitCode) { + // Check if need to do memory profiling, and do memory profiling accordingly before azcopy exits. + lcm.checkAndTriggerMemoryProfiling() + + // Check if there is ongoing CPU profiling, and stop CPU profiling. + lcm.checkAndStopCPUProfiling() + + messageContent := "" + if o != nil { + messageContent = o(lcm.outputFormat) + } + + lcm.msgQueue <- outputMessage{ + msgContent: messageContent, + msgType: eOutputMessageType.Exit(), exitCode: exitCode, } @@ -176,90 +200,144 @@ func (lcm *lifecycleMgr) SurrenderControl() { } func (lcm *lifecycleMgr) processOutputMessage() { - // when a new line needs to overwrite the current line completely - // we need to make sure that if the new line is shorter, we properly erase everything from the current line - var matchLengthWithSpaces = func(curLineLength, newLineLength int) { - if dirtyLeftover := curLineLength - newLineLength; dirtyLeftover > 0 { - for i := 0; i < dirtyLeftover; i++ { - fmt.Print(" ") - } + // this function constantly pulls out message to output + // and pass them onto the right handler based on the output format + for { + switch msgToPrint := <-lcm.msgQueue; lcm.outputFormat { + case EOutputFormat.Json(): + lcm.processJSONOutput(msgToPrint) + case EOutputFormat.Text(): + lcm.processTextOutput(msgToPrint) + case EOutputFormat.None(): + lcm.processNoneOutput(msgToPrint) + default: + panic("unimplemented output format") } } +} - // NOTE: fmt.printf is being avoided on purpose (for memory optimization) - for { - switch msgToPrint := <-lcm.msgQueue; msgToPrint.msgType { - case eMessageType.Exit(): - // simply print and quit - // if no message is intended, avoid adding new lines - if msgToPrint.msgContent != "" { - fmt.Println("\n" + msgToPrint.msgContent) - } - os.Exit(int(msgToPrint.exitCode)) +func (lcm *lifecycleMgr) processNoneOutput(msgToOutput outputMessage) { + if msgToOutput.msgType == eOutputMessageType.Exit() { + os.Exit(int(msgToOutput.exitCode)) + } else if msgToOutput.msgType == eOutputMessageType.Error() { + os.Exit(int(EExitCode.Error())) + } - case eMessageType.Progress(): - fmt.Print("\r") // return carriage back to start - fmt.Print(msgToPrint.msgContent) // print new progress + // ignore all other outputs + return +} - // it is possible that the new progress status is somehow shorter than the previous one - // in this case we must erase the left over characters from the previous progress - matchLengthWithSpaces(len(lcm.progressCache), len(msgToPrint.msgContent)) +func (lcm *lifecycleMgr) processJSONOutput(msgToOutput outputMessage) { + msgType := msgToOutput.msgType - lcm.progressCache = msgToPrint.msgContent + // omit outputs to Stderr, since these could confuse the tools integrating AzCopy + if msgType == eOutputMessageType.StdError() { + return + } else if msgType == eOutputMessageType.Prompt() { + // TODO determine how prompts work with JSON output + // right now, we return nothing so that the default behavior is trigger for the part that intended to get response + msgToOutput.inputChannel <- "" + return + } - case eMessageType.Info(): - if lcm.progressCache != "" { // a progress status is already on the last line - // print the info from the beginning on current line - fmt.Print("\r") - fmt.Print(msgToPrint.msgContent) + // simply output the json message + // we assume the msgContent is already formatted correctly + fmt.Println(GetJsonStringFromTemplate(newJsonOutputTemplate(msgType, msgToOutput.msgContent))) - // it is possible that the info is shorter than the progress status - // in this case we must erase the left over characters from the progress status - matchLengthWithSpaces(len(lcm.progressCache), len(msgToPrint.msgContent)) + // exit if needed + if msgType == eOutputMessageType.Exit() || msgType == eOutputMessageType.Error() { + os.Exit(int(msgToOutput.exitCode)) + } +} - // print the previous progress status again, so that it's on the last line - fmt.Print("\n") - fmt.Print(lcm.progressCache) - } else { - fmt.Println(msgToPrint.msgContent) +func (lcm *lifecycleMgr) processTextOutput(msgToOutput outputMessage) { + // when a new line needs to overwrite the current line completely + // we need to make sure that if the new line is shorter, we properly erase everything from the current line + var matchLengthWithSpaces = func(curLineLength, newLineLength int) { + if dirtyLeftover := curLineLength - newLineLength; dirtyLeftover > 0 { + for i := 0; i < dirtyLeftover; i++ { + fmt.Print(" ") } + } + } - case eMessageType.Error(): - // we need to print to stderr but it's mostly likely that both stdout and stderr are directed to the terminal - // in case we are already printing progress to stdout, we need to make sure that the content from - // stderr gets displayed properly on its own line - if lcm.progressCache != "" { // a progress status is already on the last line - // erase the progress status - fmt.Print("\r") - matchLengthWithSpaces(len(lcm.progressCache), 0) - fmt.Print("\r") - - os.Stderr.WriteString(msgToPrint.msgContent) - - // print the previous progress status again, so that it's on the last line - fmt.Print("\n") - fmt.Print(lcm.progressCache) - } else { - os.Stderr.WriteString(msgToPrint.msgContent) - } + switch msgToOutput.msgType { + case eOutputMessageType.Error(): + // same handling as Exit + fallthrough + case eOutputMessageType.Exit(): + // simply print and quit + // if no message is intended, avoid adding new lines + if msgToOutput.msgContent != "" { + fmt.Println("\n" + msgToOutput.msgContent) + } + os.Exit(int(msgToOutput.exitCode)) + + case eOutputMessageType.Progress(): + fmt.Print("\r") // return carriage back to start + fmt.Print(msgToOutput.msgContent) // print new progress + + // it is possible that the new progress status is somehow shorter than the previous one + // in this case we must erase the left over characters from the previous progress + matchLengthWithSpaces(len(lcm.progressCache), len(msgToOutput.msgContent)) + + lcm.progressCache = msgToOutput.msgContent + + case eOutputMessageType.Init(): + // same handling as Info + fallthrough + case eOutputMessageType.Info(): + if lcm.progressCache != "" { // a progress status is already on the last line + // print the info from the beginning on current line + fmt.Print("\r") + fmt.Print(msgToOutput.msgContent) + + // it is possible that the info is shorter than the progress status + // in this case we must erase the left over characters from the progress status + matchLengthWithSpaces(len(lcm.progressCache), len(msgToOutput.msgContent)) + + // print the previous progress status again, so that it's on the last line + fmt.Print("\n") + fmt.Print(lcm.progressCache) + } else { + fmt.Println(msgToOutput.msgContent) + } - case eMessageType.Prompt(): - if lcm.progressCache != "" { // a progress status is already on the last line - // print the prompt from the beginning on current line - fmt.Print("\r") - fmt.Print(msgToPrint.msgContent) + case eOutputMessageType.StdError(): + // we need to print to stderr but it's mostly likely that both stdout and stderr are directed to the terminal + // in case we are already printing progress to stdout, we need to make sure that the content from + // stderr gets displayed properly on its own line + if lcm.progressCache != "" { // a progress status is already on the last line + // erase the progress status + fmt.Print("\r") + matchLengthWithSpaces(len(lcm.progressCache), 0) + fmt.Print("\r") + + os.Stderr.WriteString(msgToOutput.msgContent) + + // print the previous progress status again, so that it's on the last line + fmt.Print("\n") + fmt.Print(lcm.progressCache) + } else { + os.Stderr.WriteString(msgToOutput.msgContent) + } - // it is possible that the prompt is shorter than the progress status - // in this case we must erase the left over characters from the progress status - matchLengthWithSpaces(len(lcm.progressCache), len(msgToPrint.msgContent)) + case eOutputMessageType.Prompt(): + if lcm.progressCache != "" { // a progress status is already on the last line + // print the prompt from the beginning on current line + fmt.Print("\r") + fmt.Print(msgToOutput.msgContent) - } else { - fmt.Print(msgToPrint.msgContent) - } + // it is possible that the prompt is shorter than the progress status + // in this case we must erase the left over characters from the progress status + matchLengthWithSpaces(len(lcm.progressCache), len(msgToOutput.msgContent)) - // read the response to the prompt and send it back through the channel - msgToPrint.inputChannel <- lcm.readInCleanLineFromStdIn() + } else { + fmt.Print(msgToOutput.msgContent) } + + // read the response to the prompt and send it back through the channel + msgToOutput.inputChannel <- lcm.readInCleanLineFromStdIn() } } diff --git a/common/output.go b/common/output.go new file mode 100644 index 000000000..f5eb447f6 --- /dev/null +++ b/common/output.go @@ -0,0 +1,81 @@ +package common + +import ( + "encoding/json" + "github.com/JeffreyRichter/enum/enum" + "reflect" + "strings" + "time" +) + +var eOutputMessageType = outputMessageType(0) + +// outputMessageType defines the nature of the output, ex: progress report, job summary, or error +type outputMessageType uint8 + +func (outputMessageType) Init() outputMessageType { return outputMessageType(0) } // simple print, allowed to float up +func (outputMessageType) Info() outputMessageType { return outputMessageType(1) } // simple print, allowed to float up +func (outputMessageType) Progress() outputMessageType { return outputMessageType(2) } // should be printed on the same line over and over again, not allowed to float up +func (outputMessageType) Exit() outputMessageType { return outputMessageType(3) } // exit after printing +func (outputMessageType) Error() outputMessageType { return outputMessageType(4) } // indicate fatal error, exit right after +func (outputMessageType) StdError() outputMessageType { return outputMessageType(5) } // print to stderr +func (outputMessageType) Prompt() outputMessageType { return outputMessageType(6) } // ask the user a question after erasing the progress + +func (o outputMessageType) String() string { + return enum.StringInt(o, reflect.TypeOf(o)) +} + +// defines the output and how it should be handled +type outputMessage struct { + msgContent string + msgType outputMessageType + exitCode ExitCode // only for when the application is meant to exit after printing (i.e. Error or Final) + inputChannel chan<- string // support getting a response from the user +} + +// used for output types that are not simple strings, such as progress and init +// a given format(text,json) is passed in, and the appropriate string is returned +type outputBuilder func(OutputFormat) string + +// -------------------------------------- JSON templates -------------------------------------- // +// used to help formatting of JSON outputs + +func GetJsonStringFromTemplate(template interface{}) string { + jsonOutput, err := json.Marshal(template) + PanicIfErr(err) + + return string(jsonOutput) +} + +// defines the general output template when the format is set to json +type jsonOutputTemplate struct { + TimeStamp time.Time + MessageType string + MessageContent string // a simple string for INFO and ERROR, a serialized JSON for INIT, PROGRESS, EXIT +} + +func newJsonOutputTemplate(messageType outputMessageType, messageContent string) *jsonOutputTemplate { + return &jsonOutputTemplate{TimeStamp: time.Now(), MessageType: messageType.String(), MessageContent: messageContent} +} + +type InitMsgJsonTemplate struct { + LogFileLocation string + JobID string +} + +func GetStandardInitOutputBuilder(jobID string, logFileLocation string) outputBuilder { + return func(format OutputFormat) string { + if format == EOutputFormat.Json() { + return GetJsonStringFromTemplate(InitMsgJsonTemplate{ + JobID: jobID, + LogFileLocation: logFileLocation, + }) + } + + var sb strings.Builder + sb.WriteString("\nJob " + jobID + " has started\n") + sb.WriteString("Log file is located at: " + logFileLocation) + sb.WriteString("\n") + return sb.String() + } +} diff --git a/common/rpc-models.go b/common/rpc-models.go index 57fc69a91..afb1a7a35 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -115,8 +115,8 @@ type ListContainerResponse struct { // represents the JobProgressPercentage Summary response for list command when requested the Job Progress Summary for given JobId type ListJobSummaryResponse struct { ErrorMsg string - Timestamp time.Time - JobID JobID + Timestamp time.Time `json:"-"` + JobID JobID `json:"-"` // TODO: added for debugging purpose. remove later ActiveConnections int64 // CompleteJobOrdered determines whether the Job has been completely ordered or not @@ -134,14 +134,14 @@ type ListJobSummaryResponse struct { FailedTransfers []TransferDetail SkippedTransfers []TransferDetail IsDiskConstrained bool - PerfStrings []string + PerfStrings []string `json:"-"` } // represents the JobProgressPercentage Summary response for list command when requested the Job Progress Summary for given JobId type ListSyncJobSummaryResponse struct { ErrorMsg string - Timestamp time.Time - JobID JobID + Timestamp time.Time `json:"-"` + JobID JobID `json:"-"` // TODO: added for debugging purpose. remove later ActiveConnections int64 // CompleteJobOrdered determines whether the Job has been completely ordered or not @@ -156,7 +156,7 @@ type ListSyncJobSummaryResponse struct { DeleteTransfersFailed uint32 FailedTransfers []TransferDetail IsDiskConstrained bool - PerfStrings []string + PerfStrings []string `json:"-"` // sum of the size of transfer completed successfully so far. TotalBytesTransferred uint64 // sum of the total transfer enumerated so far. diff --git a/main.go b/main.go index 4cbc345af..72846dfa9 100644 --- a/main.go +++ b/main.go @@ -55,9 +55,11 @@ func main() { configureGC() - ste.MainSTE(common.ComputeConcurrencyValue(runtime.NumCPU()), 2400, azcopyAppPathFolder, azcopyLogPathFolder) + err = ste.MainSTE(common.ComputeConcurrencyValue(runtime.NumCPU()), 2400, azcopyAppPathFolder, azcopyLogPathFolder) + common.PanicIfErr(err) + cmd.Execute(azcopyAppPathFolder, azcopyLogPathFolder) - glcm.Exit("", common.EExitCode.Success()) + glcm.Exit(nil, common.EExitCode.Success()) } // Golang's default behaviour is to GC when new objects = (100% of) total of objects surviving previous GC. diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index 98ae9771b..890becf27 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -364,7 +364,7 @@ func (jptm *jobPartTransferMgr) failActiveTransfer(typ transferErrorCode, descri // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { // TODO: should this really exit??? why not just log like everything else does??? We've Failed the transfer anyway.... - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } // TODO: right now the convention re cancellation seems to be that if you cancel, you MUST both call cancel AND diff --git a/ste/xfer-URLToBlob.go b/ste/xfer-URLToBlob.go index af8e51ede..48c052ac4 100644 --- a/ste/xfer-URLToBlob.go +++ b/ste/xfer-URLToBlob.go @@ -114,7 +114,7 @@ func URLToBlob(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } } else { @@ -262,7 +262,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } @@ -299,7 +299,7 @@ func (bbc *blockBlobCopy) generateCopyURLToBlockBlobFunc(chunkId int32, startInd // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if status == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } return } diff --git a/ste/xfer-deleteBlob.go b/ste/xfer-deleteBlob.go index 1a2b42f99..dce8c57c6 100644 --- a/ste/xfer-deleteBlob.go +++ b/ste/xfer-deleteBlob.go @@ -52,7 +52,7 @@ func DeleteBlobPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pa // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if strErr.Response().StatusCode == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } else { transferDone(common.ETransferStatus.Failed(), err) diff --git a/ste/xfer-deletefile.go b/ste/xfer-deletefile.go index 22cdf04d9..de6311394 100644 --- a/ste/xfer-deletefile.go +++ b/ste/xfer-deletefile.go @@ -52,7 +52,7 @@ func DeleteFilePrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pa // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if strErr.Response().StatusCode == http.StatusForbidden { - common.GetLifecycleMgr().Exit(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()), 1) + common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) } } else { transferDone(common.ETransferStatus.Failed(), err) From adf8924ffea8581c5278e6be2e4f1cb94f51062c Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Mon, 25 Feb 2019 00:54:03 -0800 Subject: [PATCH 52/64] Fixed tests after output changes --- cmd/zt_interceptors_for_test.go | 7 +++++-- cmd/zt_scenario_helpers_for_test.go | 1 - cmd/zt_sync_download_test.go | 1 - common/lifecyleMgr.go | 12 ++++++------ common/output.go | 4 ++-- testSuite/scripts/utility.py | 5 ++++- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/cmd/zt_interceptors_for_test.go b/cmd/zt_interceptors_for_test.go index d68c83a26..3dfcc6219 100644 --- a/cmd/zt_interceptors_for_test.go +++ b/cmd/zt_interceptors_for_test.go @@ -81,14 +81,17 @@ func (i *interceptor) reset() { // this lifecycle manager substitute does not perform any action type mockedLifecycleManager struct{} -func (mockedLifecycleManager) Progress(string) {} +func (mockedLifecycleManager) Progress(common.OutputBuilder) {} +func (mockedLifecycleManager) Init(common.OutputBuilder) {} func (mockedLifecycleManager) Info(string) {} func (mockedLifecycleManager) Prompt(string) string { return "" } -func (mockedLifecycleManager) Exit(string, common.ExitCode) {} +func (mockedLifecycleManager) Exit(common.OutputBuilder, common.ExitCode) {} func (mockedLifecycleManager) Error(string) {} +func (mockedLifecycleManager) StdError(string) {} func (mockedLifecycleManager) SurrenderControl() {} func (mockedLifecycleManager) InitiateProgressReporting(common.WorkController, bool) {} func (mockedLifecycleManager) GetEnvironmentVariable(common.EnvironmentVariable) string { return "" } +func (mockedLifecycleManager) SetOutputFormat(common.OutputFormat) {} type dummyProcessor struct { record []storedObject diff --git a/cmd/zt_scenario_helpers_for_test.go b/cmd/zt_scenario_helpers_for_test.go index d962bda9f..52a7d7fc7 100644 --- a/cmd/zt_scenario_helpers_for_test.go +++ b/cmd/zt_scenario_helpers_for_test.go @@ -239,7 +239,6 @@ func getDefaultRawInput(src, dst string) rawSyncCmdArgs { dst: dst, recursive: true, logVerbosity: defaultLogVerbosityForSync, - output: defaultOutputFormatForSync, deleteDestination: deleteDestination.String(), md5ValidationOption: common.DefaultHashValidationOption.String(), } diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 4d574c72b..551c9b3df 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -33,7 +33,6 @@ import ( const ( defaultLogVerbosityForSync = "WARNING" - defaultOutputFormatForSync = "text" ) // regular blob->file sync diff --git a/common/lifecyleMgr.go b/common/lifecyleMgr.go index fdf4f4242..11124252f 100644 --- a/common/lifecyleMgr.go +++ b/common/lifecyleMgr.go @@ -34,9 +34,9 @@ var lcm = func() (lcmgr *lifecycleMgr) { // create a public interface so that consumers outside of this package can refer to the lifecycle manager // but they would not be able to instantiate one type LifecycleMgr interface { - Init(outputBuilder) // let the user know the job has started and initial information like log location - Progress(outputBuilder) // print on the same line over and over again, not allowed to float up - Exit(outputBuilder, ExitCode) // indicates successful execution exit after printing, allow user to specify exit code + Init(OutputBuilder) // let the user know the job has started and initial information like log location + Progress(OutputBuilder) // print on the same line over and over again, not allowed to float up + Exit(OutputBuilder, ExitCode) // indicates successful execution exit after printing, allow user to specify exit code Info(string) // simple print, allowed to float up Error(string) // indicates fatal error, exit after printing, exit code is always Failed (1) Prompt(string) string // ask the user a question(after erasing the progress), then return the response @@ -109,14 +109,14 @@ func (lcm *lifecycleMgr) checkAndTriggerMemoryProfiling() { } } -func (lcm *lifecycleMgr) Init(o outputBuilder) { +func (lcm *lifecycleMgr) Init(o OutputBuilder) { lcm.msgQueue <- outputMessage{ msgContent: o(lcm.outputFormat), msgType: eOutputMessageType.Init(), } } -func (lcm *lifecycleMgr) Progress(o outputBuilder) { +func (lcm *lifecycleMgr) Progress(o OutputBuilder) { messageContent := "" if o != nil { messageContent = o(lcm.outputFormat) @@ -171,7 +171,7 @@ func (lcm *lifecycleMgr) Error(msg string) { lcm.SurrenderControl() } -func (lcm *lifecycleMgr) Exit(o outputBuilder, exitCode ExitCode) { +func (lcm *lifecycleMgr) Exit(o OutputBuilder, exitCode ExitCode) { // Check if need to do memory profiling, and do memory profiling accordingly before azcopy exits. lcm.checkAndTriggerMemoryProfiling() diff --git a/common/output.go b/common/output.go index f5eb447f6..b32e961bc 100644 --- a/common/output.go +++ b/common/output.go @@ -35,7 +35,7 @@ type outputMessage struct { // used for output types that are not simple strings, such as progress and init // a given format(text,json) is passed in, and the appropriate string is returned -type outputBuilder func(OutputFormat) string +type OutputBuilder func(OutputFormat) string // -------------------------------------- JSON templates -------------------------------------- // // used to help formatting of JSON outputs @@ -63,7 +63,7 @@ type InitMsgJsonTemplate struct { JobID string } -func GetStandardInitOutputBuilder(jobID string, logFileLocation string) outputBuilder { +func GetStandardInitOutputBuilder(jobID string, logFileLocation string) OutputBuilder { return func(format OutputFormat) string { if format == EOutputFormat.Json() { return GetJsonStringFromTemplate(InitMsgJsonTemplate{ diff --git a/testSuite/scripts/utility.py b/testSuite/scripts/utility.py index 55fb05b4d..1c0b6a286 100644 --- a/testSuite/scripts/utility.py +++ b/testSuite/scripts/utility.py @@ -8,6 +8,7 @@ import random import json from pathlib import Path +from collections import namedtuple # Command Class is used to create azcopy commands and validator commands. @@ -664,7 +665,9 @@ def parseAzcopyOutput(s): final_output = final_output + '\n' + line else: final_output = line - return final_output + + x = json.loads(final_output, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + return x.MessageContent def get_resource_name(prefix=''): return prefix + str(uuid.uuid4()).replace('-', '') From e3919eae72d76ba6ab3354b45dafbbed8650a418 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Wed, 27 Feb 2019 17:39:11 -0800 Subject: [PATCH 53/64] Simplified outputs by removing STDERR --- cmd/zt_interceptors_for_test.go | 1 - common/lifecyleMgr.go | 46 ++----------------- common/output.go | 3 +- ste/xfer-deleteBlob.go | 4 +- ...{xfer-deletefile.go => xfer-deleteFile.go} | 4 +- 5 files changed, 12 insertions(+), 46 deletions(-) rename ste/{xfer-deletefile.go => xfer-deleteFile.go} (89%) diff --git a/cmd/zt_interceptors_for_test.go b/cmd/zt_interceptors_for_test.go index 3dfcc6219..4c5cc6f45 100644 --- a/cmd/zt_interceptors_for_test.go +++ b/cmd/zt_interceptors_for_test.go @@ -87,7 +87,6 @@ func (mockedLifecycleManager) Info(string) func (mockedLifecycleManager) Prompt(string) string { return "" } func (mockedLifecycleManager) Exit(common.OutputBuilder, common.ExitCode) {} func (mockedLifecycleManager) Error(string) {} -func (mockedLifecycleManager) StdError(string) {} func (mockedLifecycleManager) SurrenderControl() {} func (mockedLifecycleManager) InitiateProgressReporting(common.WorkController, bool) {} func (mockedLifecycleManager) GetEnvironmentVariable(common.EnvironmentVariable) string { return "" } diff --git a/common/lifecyleMgr.go b/common/lifecyleMgr.go index 11124252f..6331a353e 100644 --- a/common/lifecyleMgr.go +++ b/common/lifecyleMgr.go @@ -40,7 +40,6 @@ type LifecycleMgr interface { Info(string) // simple print, allowed to float up Error(string) // indicates fatal error, exit after printing, exit code is always Failed (1) Prompt(string) string // ask the user a question(after erasing the progress), then return the response - StdError(string) // print to stderr SurrenderControl() // give up control, this should never return InitiateProgressReporting(WorkController, bool) // start writing progress with another routine GetEnvironmentVariable(EnvironmentVariable) string // get the environment variable or its default value @@ -135,13 +134,6 @@ func (lcm *lifecycleMgr) Info(msg string) { } } -func (lcm *lifecycleMgr) StdError(msg string) { - lcm.msgQueue <- outputMessage{ - msgContent: msg, - msgType: eOutputMessageType.StdError(), - } -} - func (lcm *lifecycleMgr) Prompt(msg string) string { expectedInputChannel := make(chan string, 1) lcm.msgQueue <- outputMessage{ @@ -154,6 +146,7 @@ func (lcm *lifecycleMgr) Prompt(msg string) string { return <-expectedInputChannel } +// TODO minor: consider merging with Exit func (lcm *lifecycleMgr) Error(msg string) { // Check if need to do memory profiling, and do memory profiling accordingly before azcopy exits. lcm.checkAndTriggerMemoryProfiling() @@ -230,12 +223,9 @@ func (lcm *lifecycleMgr) processNoneOutput(msgToOutput outputMessage) { func (lcm *lifecycleMgr) processJSONOutput(msgToOutput outputMessage) { msgType := msgToOutput.msgType - // omit outputs to Stderr, since these could confuse the tools integrating AzCopy - if msgType == eOutputMessageType.StdError() { - return - } else if msgType == eOutputMessageType.Prompt() { + // right now, we return nothing so that the default behavior is triggered for the part that intended to get response + if msgType == eOutputMessageType.Prompt() { // TODO determine how prompts work with JSON output - // right now, we return nothing so that the default behavior is trigger for the part that intended to get response msgToOutput.inputChannel <- "" return } @@ -262,10 +252,7 @@ func (lcm *lifecycleMgr) processTextOutput(msgToOutput outputMessage) { } switch msgToOutput.msgType { - case eOutputMessageType.Error(): - // same handling as Exit - fallthrough - case eOutputMessageType.Exit(): + case eOutputMessageType.Error(), eOutputMessageType.Exit(): // simply print and quit // if no message is intended, avoid adding new lines if msgToOutput.msgContent != "" { @@ -283,10 +270,7 @@ func (lcm *lifecycleMgr) processTextOutput(msgToOutput outputMessage) { lcm.progressCache = msgToOutput.msgContent - case eOutputMessageType.Init(): - // same handling as Info - fallthrough - case eOutputMessageType.Info(): + case eOutputMessageType.Init(), eOutputMessageType.Info(): if lcm.progressCache != "" { // a progress status is already on the last line // print the info from the beginning on current line fmt.Print("\r") @@ -302,26 +286,6 @@ func (lcm *lifecycleMgr) processTextOutput(msgToOutput outputMessage) { } else { fmt.Println(msgToOutput.msgContent) } - - case eOutputMessageType.StdError(): - // we need to print to stderr but it's mostly likely that both stdout and stderr are directed to the terminal - // in case we are already printing progress to stdout, we need to make sure that the content from - // stderr gets displayed properly on its own line - if lcm.progressCache != "" { // a progress status is already on the last line - // erase the progress status - fmt.Print("\r") - matchLengthWithSpaces(len(lcm.progressCache), 0) - fmt.Print("\r") - - os.Stderr.WriteString(msgToOutput.msgContent) - - // print the previous progress status again, so that it's on the last line - fmt.Print("\n") - fmt.Print(lcm.progressCache) - } else { - os.Stderr.WriteString(msgToOutput.msgContent) - } - case eOutputMessageType.Prompt(): if lcm.progressCache != "" { // a progress status is already on the last line // print the prompt from the beginning on current line diff --git a/common/output.go b/common/output.go index b32e961bc..ddfe52b4b 100644 --- a/common/output.go +++ b/common/output.go @@ -18,8 +18,7 @@ func (outputMessageType) Info() outputMessageType { return outputMessageType func (outputMessageType) Progress() outputMessageType { return outputMessageType(2) } // should be printed on the same line over and over again, not allowed to float up func (outputMessageType) Exit() outputMessageType { return outputMessageType(3) } // exit after printing func (outputMessageType) Error() outputMessageType { return outputMessageType(4) } // indicate fatal error, exit right after -func (outputMessageType) StdError() outputMessageType { return outputMessageType(5) } // print to stderr -func (outputMessageType) Prompt() outputMessageType { return outputMessageType(6) } // ask the user a question after erasing the progress +func (outputMessageType) Prompt() outputMessageType { return outputMessageType(5) } // ask the user a question after erasing the progress func (o outputMessageType) String() string { return enum.StringInt(o, reflect.TypeOf(o)) diff --git a/ste/xfer-deleteBlob.go b/ste/xfer-deleteBlob.go index dce8c57c6..c3509373e 100644 --- a/ste/xfer-deleteBlob.go +++ b/ste/xfer-deleteBlob.go @@ -52,7 +52,9 @@ func DeleteBlobPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pa // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if strErr.Response().StatusCode == http.StatusForbidden { - common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) + errMsg := fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()) + jptm.Log(pipeline.LogError, errMsg) + common.GetLifecycleMgr().Error(errMsg) } } else { transferDone(common.ETransferStatus.Failed(), err) diff --git a/ste/xfer-deletefile.go b/ste/xfer-deleteFile.go similarity index 89% rename from ste/xfer-deletefile.go rename to ste/xfer-deleteFile.go index de6311394..325782e7f 100644 --- a/ste/xfer-deletefile.go +++ b/ste/xfer-deleteFile.go @@ -52,7 +52,9 @@ func DeleteFilePrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pa // If the status code was 403, it means there was an authentication error and we exit. // User can resume the job if completely ordered with a new sas. if strErr.Response().StatusCode == http.StatusForbidden { - common.GetLifecycleMgr().Error(fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error())) + errMsg := fmt.Sprintf("Authentication Failed. The SAS is not correct or expired or does not have the correct permission %s", err.Error()) + jptm.Log(pipeline.LogError, errMsg) + common.GetLifecycleMgr().Error(errMsg) } } else { transferDone(common.ETransferStatus.Failed(), err) From 4edfcb7e3c504e496f00ae0010b13b59048d97fc Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Thu, 21 Feb 2019 16:32:00 -0800 Subject: [PATCH 54/64] Store src and dest roots separately from transfer paths to save space in plan file --- cmd/copy.go | 4 +++ cmd/copyEnumeratorHelper.go | 5 ++++ cmd/copyEnumeratorHelper_test.go | 51 ++++++++++++++++++++++++++++++++ cmd/syncProcessor.go | 2 ++ cmd/zc_processor.go | 25 ++++++++-------- cmd/zc_traverser_local.go | 15 +++++++--- common/rpc-models.go | 2 ++ ste/JobPartPlan.go | 39 ++++++++++++++---------- ste/JobPartPlanFileName.go | 42 ++++++++++++++++---------- 9 files changed, 136 insertions(+), 49 deletions(-) create mode 100644 cmd/copyEnumeratorHelper_test.go diff --git a/cmd/copy.go b/cmd/copy.go index 43ddf7c97..fcd08c556 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -644,6 +644,10 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { } } + // set the source and destination roots on the job part orders after they have been computed and cleaned + jobPartOrder.SourceRoot = cca.source + jobPartOrder.DestinationRoot = cca.destination + // lastPartNumber determines the last part number order send for the Job. var lastPartNumber common.PartNumber // depending on the source and destination type, we process the cp command differently diff --git a/cmd/copyEnumeratorHelper.go b/cmd/copyEnumeratorHelper.go index 13ca52ded..1c20e6203 100644 --- a/cmd/copyEnumeratorHelper.go +++ b/cmd/copyEnumeratorHelper.go @@ -16,12 +16,17 @@ import ( // addTransfer accepts a new transfer, if the threshold is reached, dispatch a job part order. func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer, cca *cookedCopyCmdArgs) error { + // Remove the source and destination roots from the path to save space in the plan files + transfer.Source = strings.TrimPrefix(transfer.Source, e.SourceRoot) + transfer.Destination = strings.TrimPrefix(transfer.Destination, e.DestinationRoot) + // dispatch the transfers once the number reaches NumOfFilesPerDispatchJobPart // we do this so that in the case of large transfer, the transfer engine can get started // while the frontend is still gathering more transfers if len(e.Transfers) == NumOfFilesPerDispatchJobPart { shuffleTransfers(e.Transfers) resp := common.CopyJobPartOrderResponse{} + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(e), &resp) if !resp.JobStarted { diff --git a/cmd/copyEnumeratorHelper_test.go b/cmd/copyEnumeratorHelper_test.go new file mode 100644 index 000000000..645db6ea8 --- /dev/null +++ b/cmd/copyEnumeratorHelper_test.go @@ -0,0 +1,51 @@ +// Copyright © 2017 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cmd + +import ( + "github.com/Azure/azure-storage-azcopy/common" + chk "gopkg.in/check.v1" +) + +type copyEnumeratorHelperTestSuite struct{} + +var _ = chk.Suite(©EnumeratorHelperTestSuite{}) + +func (s *copyEnumeratorHelperTestSuite) TestAddTransferPathRootsTrimmed(c *chk.C) { + // setup + request := common.CopyJobPartOrderRequest{ + SourceRoot: "a/b/", + DestinationRoot: "y/z/", + } + + transfer := common.CopyTransfer{ + Source: "a/b/c.txt", + Destination: "y/z/c.txt", + } + + // execute + err := addTransfer(&request, transfer, &cookedCopyCmdArgs{}) + + // assert + c.Assert(err, chk.IsNil) + c.Assert(request.Transfers[0].Source, chk.Equals, "c.txt") + c.Assert(request.Transfers[0].Destination, chk.Equals, "c.txt") +} diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index a6d511a3e..8feb6d568 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -39,6 +39,8 @@ func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) JobID: cca.jobID, CommandString: cca.commandString, FromTo: cca.fromTo, + SourceRoot: cca.source, + DestinationRoot: cca.destination, // authentication related CredentialInfo: cca.credentialInfo, diff --git a/cmd/zc_processor.go b/cmd/zc_processor.go index a0ce08dd4..2e0673a1c 100644 --- a/cmd/zc_processor.go +++ b/cmd/zc_processor.go @@ -24,7 +24,6 @@ import ( "fmt" "github.com/Azure/azure-storage-azcopy/common" "net/url" - "strings" ) type copyTransferProcessor struct { @@ -69,14 +68,22 @@ func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject storedObject) s.copyJobTemplate.PartNum++ } - sourceObjectRelativePath := s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeSourceObjectName) - destinationObjectRelativePath := s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeDestinationObjectName) + // In the case of single file transfers, relative path is empty and we must use the object name. + var source string + var destination string + if storedObject.relativePath == "" { + source = storedObject.name + destination = storedObject.name + } else { + source = storedObject.relativePath + destination = storedObject.relativePath + } // only append the transfer after we've checked and dispatched a part // so that there is at least one transfer for the final part s.copyJobTemplate.Transfers = append(s.copyJobTemplate.Transfers, common.CopyTransfer{ - Source: s.appendObjectPathToResourcePath(sourceObjectRelativePath, s.source), - Destination: s.appendObjectPathToResourcePath(destinationObjectRelativePath, s.destination), + Source: s.escapeIfNecessary(source, s.shouldEscapeSourceObjectName), + Destination: s.escapeIfNecessary(destination, s.shouldEscapeSourceObjectName), SourceSize: storedObject.size, LastModifiedTime: storedObject.lastModifiedTime, ContentMD5: storedObject.md5, @@ -92,14 +99,6 @@ func (s *copyTransferProcessor) escapeIfNecessary(path string, shouldEscape bool return path } -func (s *copyTransferProcessor) appendObjectPathToResourcePath(storedObjectPath, parentPath string) string { - if storedObjectPath == "" { - return parentPath - } - - return strings.Join([]string{parentPath, storedObjectPath}, common.AZCOPY_PATH_SEPARATOR_STRING) -} - func (s *copyTransferProcessor) dispatchFinalPart() (copyJobInitiated bool, err error) { numberOfCopyTransfers := len(s.copyJobTemplate.Transfers) diff --git a/cmd/zc_traverser_local.go b/cmd/zc_traverser_local.go index cbbb5230c..5be2714c7 100644 --- a/cmd/zc_traverser_local.go +++ b/cmd/zc_traverser_local.go @@ -66,9 +66,8 @@ func (t *localTraverser) traverse(processor objectProcessor, filters []objectFil t.incrementEnumerationCounter() return processIfPassedFilters(filters, newStoredObject(fileInfo.Name(), - strings.Replace(filePath, t.fullPath+common.AZCOPY_PATH_SEPARATOR_STRING, "", 1), - fileInfo.ModTime(), - fileInfo.Size(), nil), processor) + strings.Replace(replacePathSeparators(filePath), t.fullPath+common.AZCOPY_PATH_SEPARATOR_STRING, + "", 1), fileInfo.ModTime(), fileInfo.Size(), nil), processor) }) return @@ -98,6 +97,14 @@ func (t *localTraverser) traverse(processor objectProcessor, filters []objectFil return } +func replacePathSeparators(path string) string { + if os.PathSeparator != common.AZCOPY_PATH_SEPARATOR_CHAR { + return strings.Replace(path, string(os.PathSeparator), common.AZCOPY_PATH_SEPARATOR_STRING, -1) + } else { + return path + } +} + func (t *localTraverser) getInfoIfSingleFile() (os.FileInfo, bool, error) { fileInfo, err := os.Stat(t.fullPath) @@ -114,7 +121,7 @@ func (t *localTraverser) getInfoIfSingleFile() (os.FileInfo, bool, error) { func newLocalTraverser(fullPath string, recursive bool, incrementEnumerationCounter func()) *localTraverser { traverser := localTraverser{ - fullPath: fullPath, + fullPath: replacePathSeparators(fullPath), recursive: recursive, incrementEnumerationCounter: incrementEnumerationCounter} return &traverser diff --git a/common/rpc-models.go b/common/rpc-models.go index afb1a7a35..38067cf7a 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -52,6 +52,8 @@ type CopyJobPartOrderRequest struct { Exclude map[string]int // list of blobTypes to exclude. ExcludeBlobType []azblob.BlobType + SourceRoot string + DestinationRoot string Transfers []CopyTransfer LogLevel LogLevel BlobAttributes BlobTransferAttributes diff --git a/ste/JobPartPlan.go b/ste/JobPartPlan.go index 5b86ee8b2..7d343b7bb 100644 --- a/ste/JobPartPlan.go +++ b/ste/JobPartPlan.go @@ -14,7 +14,7 @@ import ( // dataSchemaVersion defines the data schema version of JobPart order files supported by // current version of azcopy // To be Incremented every time when we release azcopy with changed dataSchema -const DataSchemaVersion common.Version = 2 +const DataSchemaVersion common.Version = 3 const ( ContentTypeMaxBytes = 256 // If > 65536, then jobPartPlanBlobData's ContentTypeLength's type field must change @@ -39,20 +39,24 @@ func (mmf *JobPartPlanMMF) Unmap() { (*common.MMF)(mmf).Unmap() } // JobPartPlanHeader represents the header of Job Part's memory-mapped file type JobPartPlanHeader struct { // Once set, the following fields are constants; they should never be modified - Version common.Version // The version of data schema format of header; see the dataSchemaVersion constant - StartTime int64 // The start time of this part - JobID common.JobID // Job Part's JobID - PartNum common.PartNumber // Job Part's part number (0+) - IsFinalPart bool // True if this is the Job's last part; else false - ForceWrite bool // True if the existing blobs needs to be overwritten. - Priority common.JobPriority // The Job Part's priority - TTLAfterCompletion uint32 // Time to live after completion is used to persists the file on disk of specified time after the completion of JobPartOrder - FromTo common.FromTo // The location of the transfer's source & destination - CommandStringLength uint32 - NumTransfers uint32 // The number of transfers in the Job part - LogLevel common.LogLevel // This Job Part's minimal log level - DstBlobData JobPartPlanDstBlob // Additional data for blob destinations - DstLocalData JobPartPlanDstLocal // Additional data for local destinations + Version common.Version // The version of data schema format of header; see the dataSchemaVersion constant + StartTime int64 // The start time of this part + JobID common.JobID // Job Part's JobID + PartNum common.PartNumber // Job Part's part number (0+) + SourceRootLength uint16 // The length of the source root path + SourceRoot [1000]byte // The root directory of the source + DestinationRootLength uint16 // The length of the destination root path + DestinationRoot [1000]byte // The root directory of the destination + IsFinalPart bool // True if this is the Job's last part; else false + ForceWrite bool // True if the existing blobs needs to be overwritten. + Priority common.JobPriority // The Job Part's priority + TTLAfterCompletion uint32 // Time to live after completion is used to persists the file on disk of specified time after the completion of JobPartOrder + FromTo common.FromTo // The location of the transfer's source & destination + CommandStringLength uint32 + NumTransfers uint32 // The number of transfers in the Job part + LogLevel common.LogLevel // This Job Part's minimal log level + DstBlobData JobPartPlanDstBlob // Additional data for blob destinations + DstLocalData JobPartPlanDstLocal // Additional data for local destinations // Any fields below this comment are NOT constants; they may change over as the job part is processed. // Care must be taken to read/write to these fields in a thread-safe way! @@ -97,6 +101,9 @@ func (jpph *JobPartPlanHeader) CommandString() string { // TransferSrcDstDetail returns the source and destination string for a transfer at given transferIndex in JobPartOrder func (jpph *JobPartPlanHeader) TransferSrcDstStrings(transferIndex uint32) (source, destination string) { + srcRoot := string(jpph.SourceRoot[:jpph.SourceRootLength]) + dstRoot := string(jpph.DestinationRoot[:jpph.DestinationRootLength]) + jppt := jpph.Transfer(transferIndex) srcSlice := []byte{} @@ -111,7 +118,7 @@ func (jpph *JobPartPlanHeader) TransferSrcDstStrings(transferIndex uint32) (sour sh.Len = int(jppt.DstLength) sh.Cap = sh.Len - return string(srcSlice), string(dstSlice) + return srcRoot + string(srcSlice), dstRoot + string(dstSlice) } func (jpph *JobPartPlanHeader) getString(offset int64, length int16) string { diff --git a/ste/JobPartPlanFileName.go b/ste/JobPartPlanFileName.go index 5422b843f..0468f807e 100644 --- a/ste/JobPartPlanFileName.go +++ b/ste/JobPartPlanFileName.go @@ -70,14 +70,20 @@ func (jpfn JobPartPlanFileName) Map() *JobPartPlanMMF { // createJobPartPlanFile creates the memory map JobPartPlanHeader using the given JobPartOrder and JobPartPlanBlobData func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { // Validate that the passed-in strings can fit in their respective fields + if len(order.SourceRoot) > len(JobPartPlanHeader{}.SourceRoot) { + panic(fmt.Errorf("source root string is too large: %q", order.SourceRoot)) + } + if len(order.DestinationRoot) > len(JobPartPlanHeader{}.DestinationRoot) { + panic(fmt.Errorf("destination root string is too large: %q", order.DestinationRoot)) + } if len(order.BlobAttributes.ContentType) > len(JobPartPlanDstBlob{}.ContentType) { - panic(fmt.Errorf("content type string it too large: %q", order.BlobAttributes.ContentType)) + panic(fmt.Errorf("content type string is too large: %q", order.BlobAttributes.ContentType)) } if len(order.BlobAttributes.ContentEncoding) > len(JobPartPlanDstBlob{}.ContentEncoding) { - panic(fmt.Errorf("content encoding string it too large: %q", order.BlobAttributes.ContentEncoding)) + panic(fmt.Errorf("content encoding string is too large: %q", order.BlobAttributes.ContentEncoding)) } if len(order.BlobAttributes.Metadata) > len(JobPartPlanDstBlob{}.Metadata) { - panic(fmt.Errorf("metadata string it too large: %q", order.BlobAttributes.Metadata)) + panic(fmt.Errorf("metadata string is too large: %q", order.BlobAttributes.Metadata)) } // This nested function writes a structure value to an io.Writer & returns the number of bytes written @@ -131,18 +137,20 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { //} // Initialize the Job Part's Plan header jpph := JobPartPlanHeader{ - Version: DataSchemaVersion, - StartTime: time.Now().UnixNano(), - JobID: order.JobID, - PartNum: order.PartNum, - IsFinalPart: order.IsFinalPart, - ForceWrite: order.ForceWrite, - Priority: order.Priority, - TTLAfterCompletion: uint32(time.Time{}.Nanosecond()), - FromTo: order.FromTo, - CommandStringLength: uint32(len(order.CommandString)), - NumTransfers: uint32(len(order.Transfers)), - LogLevel: order.LogLevel, + Version: DataSchemaVersion, + StartTime: time.Now().UnixNano(), + JobID: order.JobID, + PartNum: order.PartNum, + SourceRootLength: uint16(len(order.SourceRoot)), + DestinationRootLength: uint16(len(order.DestinationRoot)), + IsFinalPart: order.IsFinalPart, + ForceWrite: order.ForceWrite, + Priority: order.Priority, + TTLAfterCompletion: uint32(time.Time{}.Nanosecond()), + FromTo: order.FromTo, + CommandStringLength: uint32(len(order.CommandString)), + NumTransfers: uint32(len(order.Transfers)), + LogLevel: order.LogLevel, DstBlobData: JobPartPlanDstBlob{ BlobType: order.BlobAttributes.BlobType, NoGuessMimeType: order.BlobAttributes.NoGuessMimeType, @@ -161,6 +169,8 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { } // Copy any strings into their respective fields + copy(jpph.SourceRoot[:], order.SourceRoot) + copy(jpph.DestinationRoot[:], order.DestinationRoot) copy(jpph.DstBlobData.ContentType[:], order.BlobAttributes.ContentType) copy(jpph.DstBlobData.ContentEncoding[:], order.BlobAttributes.ContentEncoding) copy(jpph.DstBlobData.Metadata[:], order.BlobAttributes.Metadata) @@ -226,7 +236,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) { jppt.SrcCacheControlLength + jppt.SrcContentMD5Length + jppt.SrcMetadataLength + jppt.SrcBlobTypeLength) } - // All the transfers were written; now write each each transfer's src/dst strings + // All the transfers were written; now write each transfer's src/dst strings for t := range order.Transfers { // Sanity check: Verify that we are were we think we are and that no bug has occurred if eof != srcDstStringsOffset[t] { From c0493fa08921098d1a231b60bacc1f200533a4f5 Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Fri, 22 Feb 2019 11:27:38 -0800 Subject: [PATCH 55/64] Updated tests --- cmd/zc_processor.go | 15 ++------------- cmd/zt_generic_processor_test.go | 12 +++++++++--- cmd/zt_sync_download_test.go | 3 ++- cmd/zt_sync_upload_test.go | 3 ++- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/zc_processor.go b/cmd/zc_processor.go index 2e0673a1c..d6ea6f498 100644 --- a/cmd/zc_processor.go +++ b/cmd/zc_processor.go @@ -68,22 +68,11 @@ func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject storedObject) s.copyJobTemplate.PartNum++ } - // In the case of single file transfers, relative path is empty and we must use the object name. - var source string - var destination string - if storedObject.relativePath == "" { - source = storedObject.name - destination = storedObject.name - } else { - source = storedObject.relativePath - destination = storedObject.relativePath - } - // only append the transfer after we've checked and dispatched a part // so that there is at least one transfer for the final part s.copyJobTemplate.Transfers = append(s.copyJobTemplate.Transfers, common.CopyTransfer{ - Source: s.escapeIfNecessary(source, s.shouldEscapeSourceObjectName), - Destination: s.escapeIfNecessary(destination, s.shouldEscapeSourceObjectName), + Source: s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeSourceObjectName), + Destination: s.escapeIfNecessary(storedObject.relativePath, s.shouldEscapeDestinationObjectName), SourceSize: storedObject.size, LastModifiedTime: storedObject.lastModifiedTime, ContentMD5: storedObject.md5, diff --git a/cmd/zt_generic_processor_test.go b/cmd/zt_generic_processor_test.go index 32705eaa5..611053d64 100644 --- a/cmd/zt_generic_processor_test.go +++ b/cmd/zt_generic_processor_test.go @@ -121,8 +121,9 @@ func (s *genericProcessorSuite) TestCopyTransferProcessorSingleFile(c *chk.C) { mockedRPC.init() // set up the processor + blobURL := containerURL.NewBlockBlobURL(blobList[0]).String() copyProcessor := newCopyTransferProcessor(processorTestSuiteHelper{}.getCopyJobTemplate(), 2, - containerURL.NewBlockBlobURL(blobList[0]).String(), filepath.Join(dstDirName, dstFileName), false, false, nil, nil) + blobURL, filepath.Join(dstDirName, dstFileName), false, false, nil, nil) // exercise the copy transfer processor storedObject := newStoredObject(blobList[0], "", time.Now(), 0, nil) @@ -136,7 +137,12 @@ func (s *genericProcessorSuite) TestCopyTransferProcessorSingleFile(c *chk.C) { jobInitiated, err := copyProcessor.dispatchFinalPart() c.Assert(jobInitiated, chk.Equals, true) + // In cases of syncing file to file, the source and destination are empty because this info is already in the root + // path. + c.Assert(mockedRPC.transfers[0].Source, chk.Equals, "") + c.Assert(mockedRPC.transfers[0].Destination, chk.Equals, "") + // assert the right transfers were scheduled - validateTransfersAreScheduled(c, containerURL.String(), false, dstDirName, false, - blobList, mockedRPC) + validateTransfersAreScheduled(c, blobURL, false, filepath.Join(dstDirName, dstFileName), false, + []string{""}, mockedRPC) } diff --git a/cmd/zt_sync_download_test.go b/cmd/zt_sync_download_test.go index 551c9b3df..929cef767 100644 --- a/cmd/zt_sync_download_test.go +++ b/cmd/zt_sync_download_test.go @@ -76,7 +76,8 @@ func (s *cmdIntegrationSuite) TestSyncDownloadWithSingleFile(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateDownloadTransfersAreScheduled(c, containerURL.String(), dstDirName, blobList, mockedRPC) + validateDownloadTransfersAreScheduled(c, containerURL.NewBlobURL(blobName).String(), + filepath.Join(dstDirName, dstFileName), []string{""}, mockedRPC) }) } } diff --git a/cmd/zt_sync_upload_test.go b/cmd/zt_sync_upload_test.go index 7b38cbcda..ab25b1459 100644 --- a/cmd/zt_sync_upload_test.go +++ b/cmd/zt_sync_upload_test.go @@ -71,7 +71,8 @@ func (s *cmdIntegrationSuite) TestSyncUploadWithSingleFile(c *chk.C) { runSyncAndVerify(c, raw, func(err error) { c.Assert(err, chk.IsNil) - validateUploadTransfersAreScheduled(c, srcDirName, containerURL.String(), fileList, mockedRPC) + validateUploadTransfersAreScheduled(c, filepath.Join(srcDirName, srcFileName), + containerURL.NewBlobURL(dstBlobName).String(), []string{""}, mockedRPC) }) } } From 45d069575bdca5ae83160487229c363b5c542f3e Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Thu, 14 Feb 2019 17:54:55 -0800 Subject: [PATCH 56/64] Transfer will now fail if the source is modified after the transfer is scheduled --- ste/downloader-azureFiles.go | 7 +++++++ ste/downloader-blob.go | 4 +++- ste/downloader-blobFS.go | 7 +++++++ ste/mgr-JobPartTransferMgr.go | 5 +++++ ste/xfer-localToRemote.go | 26 ++++++++++++++++++++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/ste/downloader-azureFiles.go b/ste/downloader-azureFiles.go index 19e434de3..76676ab5e 100644 --- a/ste/downloader-azureFiles.go +++ b/ste/downloader-azureFiles.go @@ -21,6 +21,7 @@ package ste import ( + "errors" "net/url" "github.com/Azure/azure-pipeline-go/pipeline" @@ -53,6 +54,12 @@ func (bd *azureFilesDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, s return } + // Verify that the file has not been changed via a client side LMT check + if get.LastModified() != jptm.LastModifiedTime() { + jptm.FailActiveDownload("Azure File modified during transfer", + errors.New("Azure File modified during transfer")) + } + // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) diff --git a/ste/downloader-blob.go b/ste/downloader-blob.go index 2ab9c45af..bebe0f404 100644 --- a/ste/downloader-blob.go +++ b/ste/downloader-blob.go @@ -45,7 +45,9 @@ func (bd *blobDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPipe // wait until we get the headers back... but we have not yet read its whole body. // The Download method encapsulates any retries that may be necessary to get to the point of receiving response headers. jptm.LogChunkStatus(id, common.EWaitReason.HeaderResponse()) - get, err := srcBlobURL.Download(jptm.Context(), id.OffsetInFile, length, azblob.BlobAccessConditions{}, false) + get, err := srcBlobURL.Download(jptm.Context(), id.OffsetInFile, length, + azblob.BlobAccessConditions{ModifiedAccessConditions: + azblob.ModifiedAccessConditions{IfUnmodifiedSince:jptm.LastModifiedTime()}}, false) if err != nil { jptm.FailActiveDownload("Downloading response body", err) // cancel entire transfer because this chunk has failed return diff --git a/ste/downloader-blobFS.go b/ste/downloader-blobFS.go index a99e91ac7..d323bb266 100644 --- a/ste/downloader-blobFS.go +++ b/ste/downloader-blobFS.go @@ -21,6 +21,7 @@ package ste import ( + "errors" "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/azbfs" "github.com/Azure/azure-storage-azcopy/common" @@ -52,6 +53,12 @@ func (bd *blobFSDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPi return } + // Verify that the file has not been changed via a client side LMT check + if get.LastModified() != jptm.LastModifiedTime().String() { + jptm.FailActiveDownload("BFS File modified during transfer", + errors.New("BFS File modified during transfer")) + } + // step 2: Enqueue the response body to be written out to disk // The retryReader encapsulates any retries that may be necessary while downloading the body jptm.LogChunkStatus(id, common.EWaitReason.Body()) diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index 890becf27..28f72f000 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -21,6 +21,7 @@ type IJobPartTransferMgr interface { Info() TransferInfo BlobDstData(dataFileToXfer []byte) (headers azblob.BlobHTTPHeaders, metadata azblob.Metadata) FileDstData(dataFileToXfer []byte) (headers azfile.FileHTTPHeaders, metadata azfile.Metadata) + LastModifiedTime() time.Time PreserveLastModifiedTime() (time.Time, bool) MD5ValidationOption() common.HashValidationOption BlobTiers() (blockBlobTier common.BlockBlobTier, pageBlobTier common.PageBlobTier) @@ -217,6 +218,10 @@ func (jptm *jobPartTransferMgr) FileDstData(dataFileToXfer []byte) (headers azfi return jptm.jobPartMgr.(*jobPartMgr).fileDstData(jptm.Info().Source, dataFileToXfer) } +func (jptm * jobPartTransferMgr) LastModifiedTime() time.Time { + return time.Unix(0, jptm.jobPartPlanTransfer.ModifiedTime) +} + // PreserveLastModifiedTime checks for the PreserveLastModifiedTime flag in JobPartPlan of a transfer. // If PreserveLastModifiedTime is set to true, it returns the lastModifiedTime of the source. func (jptm *jobPartTransferMgr) PreserveLastModifiedTime() (time.Time, bool) { diff --git a/ste/xfer-localToRemote.go b/ste/xfer-localToRemote.go index d02cb61dc..850c262dc 100644 --- a/ste/xfer-localToRemote.go +++ b/ste/xfer-localToRemote.go @@ -22,6 +22,7 @@ package ste import ( "crypto/md5" + "errors" "fmt" "os" @@ -94,6 +95,20 @@ func localToRemote(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer, } defer srcFile.Close() // we read all the chunks in this routine, so can close the file at the end + i, err := os.Stat(info.Source) + if err != nil { + jptm.LogUploadError(info.Source, info.Destination, "Couldn't stat source-"+err.Error(), 0) + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() + return + } + if i.ModTime() != jptm.LastModifiedTime() { + jptm.LogUploadError(info.Source, info.Destination, "File modified since transfer scheduled", 0) + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() + return + } + // ***** // Error-handling rules change here. // ABOVE this point, we end the transfer using the code as shown above @@ -216,6 +231,17 @@ func isDummyChunkInEmptyFile(startIndex int64, fileSize int64) bool { // depend on the destination type func epilogueWithCleanupUpload(jptm IJobPartTransferMgr, ul uploader) { + if jptm.TransferStatus() > 0 { + // Stat the file again to see if it was changed during transfer. If it was, mark the transfer as failed. + i, err := os.Stat(jptm.Info().Source) + if err != nil { + jptm.FailActiveUpload("epilogueWithCleanupUpload", err) + } + if i.ModTime() != jptm.LastModifiedTime() { + jptm.FailActiveUpload("epilogueWithCleanupUpload", errors.New("source modified during transfer")) + } + } + ul.Epilogue() // TODO: finalize and wrap in functions whether 0 is included or excluded in status comparisons From 5a48f942259cd22150049965ac1d29b2e96d34a1 Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Tue, 26 Feb 2019 12:23:55 -0800 Subject: [PATCH 57/64] Bug fix for lmt on sync commands --- cmd/syncProcessor.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index 8feb6d568..2d66cd47b 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -36,11 +36,11 @@ import ( // extract the right info from cooked arguments and instantiate a generic copy transfer processor from it func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) *copyTransferProcessor { copyJobTemplate := &common.CopyJobPartOrderRequest{ - JobID: cca.jobID, - CommandString: cca.commandString, - FromTo: cca.fromTo, - SourceRoot: cca.source, - DestinationRoot: cca.destination, + JobID: cca.jobID, + CommandString: cca.commandString, + FromTo: cca.fromTo, + SourceRoot: replacePathSeparators(cca.source), + DestinationRoot: replacePathSeparators(cca.destination), // authentication related CredentialInfo: cca.credentialInfo, From ca62d760f78f03ef99eee29214425a00c8f5bf4a Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Wed, 27 Feb 2019 12:36:08 -0800 Subject: [PATCH 58/64] Fixed a bug with wild cards in plan file optimization --- cmd/copy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index fcd08c556..b0fb90bc8 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -645,8 +645,8 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { } // set the source and destination roots on the job part orders after they have been computed and cleaned - jobPartOrder.SourceRoot = cca.source - jobPartOrder.DestinationRoot = cca.destination + jobPartOrder.SourceRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.source) + jobPartOrder.DestinationRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.destination) // lastPartNumber determines the last part number order send for the Job. var lastPartNumber common.PartNumber From 085a1bcaf0e56c1152cdda0e5b31bdcac65b6b9e Mon Sep 17 00:00:00 2001 From: rickle-msft Date: Wed, 27 Feb 2019 17:37:52 -0800 Subject: [PATCH 59/64] Fixed bug in comparing File lmts --- ste/downloader-azureFiles.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ste/downloader-azureFiles.go b/ste/downloader-azureFiles.go index 76676ab5e..04522043b 100644 --- a/ste/downloader-azureFiles.go +++ b/ste/downloader-azureFiles.go @@ -55,7 +55,8 @@ func (bd *azureFilesDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, s } // Verify that the file has not been changed via a client side LMT check - if get.LastModified() != jptm.LastModifiedTime() { + getLocation := get.LastModified().Location() + if !get.LastModified().Equal(jptm.LastModifiedTime().In(getLocation)) { jptm.FailActiveDownload("Azure File modified during transfer", errors.New("Azure File modified during transfer")) } From fc109191d74c7a0aa34b985691a3969a8e33975c Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Wed, 27 Feb 2019 13:46:23 -0800 Subject: [PATCH 60/64] Fixed getting source/destination root information in the copy enumerator --- cmd/copy.go | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index b0fb90bc8..59b27dd4a 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -568,6 +568,7 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { } } + // TODO remove this copy pasted code during refactoring from := cca.fromTo.From() to := cca.fromTo.To() // Strip the SAS from the source and destination whenever there is SAS exists in URL. @@ -584,6 +585,11 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { blobParts.SAS = azblob.SASQueryParameters{} bUrl := blobParts.URL() cca.source = bUrl.String() + + // set the clean source root + bUrl.Path, _ = gCopyUtil.getRootPathWithoutWildCards(bUrl.Path) + jobPartOrder.SourceRoot = bUrl.String() + case common.ELocation.File(): fromUrl, err := url.Parse(cca.source) if err != nil { @@ -595,6 +601,25 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { fileParts.SAS = azfile.SASQueryParameters{} fUrl := fileParts.URL() cca.source = fUrl.String() + + // set the clean source root + fUrl.Path, _ = gCopyUtil.getRootPathWithoutWildCards(fUrl.Path) + jobPartOrder.SourceRoot = fUrl.String() + + case common.ELocation.Local(): + // If the path separator is '\\', it means + // local path is a windows path + // To avoid path separator check and handling the windows + // path differently, replace the path separator with the + // the linux path separator '/' + if os.PathSeparator == '\\' { + cca.source = strings.Replace(cca.source, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) + } + + jobPartOrder.SourceRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.source) + + default: + jobPartOrder.SourceRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.source) } switch to { @@ -620,33 +645,19 @@ func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { fileParts.SAS = azfile.SASQueryParameters{} fUrl := fileParts.URL() cca.destination = fUrl.String() - } - - if from == common.ELocation.Local() { - // If the path separator is '\\', it means - // local path is a windows path - // To avoid path separator check and handling the windows - // path differently, replace the path separator with the - // the linux path separator '/' - if os.PathSeparator == '\\' { - cca.source = strings.Replace(cca.source, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) - } - } - - if to == common.ELocation.Local() { + case common.ELocation.Local(): // If the path separator is '\\', it means // local path is a windows path // To avoid path separator check and handling the windows // path differently, replace the path separator with the // the linux path separator '/' if os.PathSeparator == '\\' { - cca.destination = strings.Replace(cca.destination, common.OS_PATH_SEPARATOR, "/", -1) + cca.destination = strings.Replace(cca.destination, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1) } } - // set the source and destination roots on the job part orders after they have been computed and cleaned - jobPartOrder.SourceRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.source) - jobPartOrder.DestinationRoot, _ = gCopyUtil.getRootPathWithoutWildCards(cca.destination) + // set the root destination after it's been cleaned + jobPartOrder.DestinationRoot = cca.destination // lastPartNumber determines the last part number order send for the Job. var lastPartNumber common.PartNumber From 3560281b7c2cace1dc4c664ef0a2a5bcbd42e7b0 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Wed, 27 Feb 2019 23:45:12 -0800 Subject: [PATCH 61/64] Fix BlobFS lmt check --- ste/downloader-blobFS.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ste/downloader-blobFS.go b/ste/downloader-blobFS.go index d323bb266..2b7bde909 100644 --- a/ste/downloader-blobFS.go +++ b/ste/downloader-blobFS.go @@ -26,6 +26,7 @@ import ( "github.com/Azure/azure-storage-azcopy/azbfs" "github.com/Azure/azure-storage-azcopy/common" "net/url" + "time" ) type blobFSDownloader struct{} @@ -53,8 +54,13 @@ func (bd *blobFSDownloader) GenerateDownloadFunc(jptm IJobPartTransferMgr, srcPi return } + // parse the remote lmt, there shouldn't be any error, unless the service returned a new format + remoteLastModified, err := time.Parse(time.RFC1123, get.LastModified()) + common.PanicIfErr(err) + remoteLmtLocation := remoteLastModified.Location() + // Verify that the file has not been changed via a client side LMT check - if get.LastModified() != jptm.LastModifiedTime().String() { + if !remoteLastModified.Equal(jptm.LastModifiedTime().In(remoteLmtLocation)) { jptm.FailActiveDownload("BFS File modified during transfer", errors.New("BFS File modified during transfer")) } From 8155486ec98f0867c64a42c2cada4950222d665b Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 1 Mar 2019 00:34:02 -0800 Subject: [PATCH 62/64] Fixed getting source/destination root information in the sync enumerator --- cmd/sync.go | 6 ++++-- cmd/syncEnumerator.go | 4 ++-- cmd/syncProcessor.go | 7 ++++++- ste/mgr-JobPartTransferMgr.go | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/sync.go b/cmd/sync.go index 8fedf387d..45f8895b3 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -24,6 +24,7 @@ import ( "context" "encoding/json" "fmt" + "path" "time" "net/url" @@ -80,9 +81,10 @@ func (raw *rawSyncCmdArgs) separateSasFromURL(rawURL string) (cleanURL string, s blobParts := azblob.NewBlobURLParts(*fromUrl) sas = blobParts.SAS.Encode() - // get clean URL without SAS + // get clean URL without SAS and trailing / in the path blobParts.SAS = azblob.SASQueryParameters{} bUrl := blobParts.URL() + bUrl.Path = strings.TrimSuffix(bUrl.Path, common.AZCOPY_PATH_SEPARATOR_STRING) cleanURL = bUrl.String() return @@ -97,7 +99,7 @@ func (raw *rawSyncCmdArgs) cleanLocalPath(rawPath string) (cleanPath string) { if os.PathSeparator == '\\' { cleanPath = strings.Replace(rawPath, common.OS_PATH_SEPARATOR, "/", -1) } - cleanPath = rawPath + cleanPath = path.Clean(rawPath) return } diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 03170c522..810a3ad50 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -50,7 +50,7 @@ func newSyncDownloadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerat return nil, errors.New("sync must happen between source and destination of the same type: either blob <-> file, or container/virtual directory <-> local directory") } - transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart) + transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart, isSingleBlob && isSingleFile) includeFilters := buildIncludeFilters(cca.include) excludeFilters := buildExcludeFilters(cca.exclude) @@ -109,7 +109,7 @@ func newSyncUploadEnumerator(cca *cookedSyncCmdArgs) (enumerator *syncEnumerator return nil, errors.New("sync must happen between source and destination of the same type: either blob <-> file, or container/virtual directory <-> local directory") } - transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart) + transferScheduler := newSyncTransferProcessor(cca, NumOfFilesPerDispatchJobPart, isSingleBlob && isSingleFile) includeFilters := buildIncludeFilters(cca.include) excludeFilters := buildExcludeFilters(cca.exclude) diff --git a/cmd/syncProcessor.go b/cmd/syncProcessor.go index 2d66cd47b..5f9dc8cce 100644 --- a/cmd/syncProcessor.go +++ b/cmd/syncProcessor.go @@ -34,7 +34,7 @@ import ( ) // extract the right info from cooked arguments and instantiate a generic copy transfer processor from it -func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) *copyTransferProcessor { +func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int, isSingleFileSync bool) *copyTransferProcessor { copyJobTemplate := &common.CopyJobPartOrderRequest{ JobID: cca.jobID, CommandString: cca.commandString, @@ -56,6 +56,11 @@ func newSyncTransferProcessor(cca *cookedSyncCmdArgs, numOfTransfersPerPart int) LogLevel: cca.logVerbosity, } + if !isSingleFileSync { + copyJobTemplate.SourceRoot += common.AZCOPY_PATH_SEPARATOR_STRING + copyJobTemplate.DestinationRoot += common.AZCOPY_PATH_SEPARATOR_STRING + } + reportFirstPart := func() { cca.setFirstPartOrdered() } reportFinalPart := func() { cca.isEnumerationComplete = true } diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index 28f72f000..b5332f7a2 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -218,7 +218,8 @@ func (jptm *jobPartTransferMgr) FileDstData(dataFileToXfer []byte) (headers azfi return jptm.jobPartMgr.(*jobPartMgr).fileDstData(jptm.Info().Source, dataFileToXfer) } -func (jptm * jobPartTransferMgr) LastModifiedTime() time.Time { +// TODO refactor into something like jptm.IsLastModifiedTimeEqual() so that there is NO LastModifiedTime method and people therefore CAN'T do it wrong due to time zone +func (jptm *jobPartTransferMgr) LastModifiedTime() time.Time { return time.Unix(0, jptm.jobPartPlanTransfer.ModifiedTime) } From ba92fe20058dc5e28ca8c201d677b25b89018a01 Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 1 Mar 2019 01:08:34 -0800 Subject: [PATCH 63/64] Added sync smoke tests --- testSuite/scripts/run.py | 3 + testSuite/scripts/test_blob_sync.py | 85 +++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 testSuite/scripts/test_blob_sync.py diff --git a/testSuite/scripts/run.py b/testSuite/scripts/run.py index 529b26bac..9914f40f2 100644 --- a/testSuite/scripts/run.py +++ b/testSuite/scripts/run.py @@ -9,6 +9,7 @@ from test_blobfs_download_sharedkey import * from test_blobfs_download_oauth import * from test_blob_piping import * +from test_blob_sync import * from test_service_to_service_copy import * import glob, os import configparser @@ -16,6 +17,7 @@ import sys import unittest + def parse_config_file_set_env(): config = configparser.RawConfigParser() files_read = config.read('../test_suite_config.ini') @@ -163,6 +165,7 @@ def main(): init() test_class_to_run = [BlobPipingTests, + Blob_Sync_User_Scenario, Block_Upload_User_Scenarios, Blob_Download_User_Scenario, PageBlob_Upload_User_Scenarios, diff --git a/testSuite/scripts/test_blob_sync.py b/testSuite/scripts/test_blob_sync.py new file mode 100644 index 000000000..7542fe4ca --- /dev/null +++ b/testSuite/scripts/test_blob_sync.py @@ -0,0 +1,85 @@ +import json +import os +import shutil +import time +import urllib +from collections import namedtuple +import utility as util +import unittest + + +# Temporary tests (mostly copy-pasted from blob tests) to guarantee simple sync scenarios still work +# TODO Replace with better tests in the future +class Blob_Sync_User_Scenario(unittest.TestCase): + + def test_sync_single_blob(self): + # create file of size 1KB. + filename = "test_1kb_blob_sync.txt" + file_path = util.create_test_file(filename, 1024) + blob_path = util.get_resource_sas(filename) + + # Upload 1KB file using azcopy. + src = file_path + dest = blob_path + result = util.Command("cp").add_arguments(src).add_arguments(dest). \ + add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + # Verifying the uploaded blob. + # the resource local path should be the first argument for the azcopy validator. + # the resource sas should be the second argument for azcopy validator. + resource_url = util.get_resource_sas(filename) + result = util.Command("testBlob").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify() + self.assertTrue(result) + + # Sync 1KB file to local using azcopy. + src = blob_path + dest = file_path + result = util.Command("sync").add_arguments(src).add_arguments(dest). \ + add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + # Sync 1KB file to blob using azcopy. + # reset local file lmt first + util.create_test_file(filename, 1024) + src = file_path + dest = blob_path + result = util.Command("sync").add_arguments(src).add_arguments(dest). \ + add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + def test_sync_entire_directory(self): + dir_name = "dir_sync_test" + dir_path = util.create_test_n_files(1024, 10, dir_name) + + # create sub-directory inside directory + sub_dir_name = os.path.join(dir_name, "sub_dir_sync_test") + util.create_test_n_files(1024, 10, sub_dir_name) + + # upload the directory with 20 files + # upload the directory + # execute azcopy command + result = util.Command("copy").add_arguments(dir_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() + self.assertTrue(result) + + # execute the validator. + vdir_sas = util.get_resource_sas(dir_name) + result = util.Command("testBlob").add_arguments(dir_path).add_arguments(vdir_sas). \ + add_flags("is-object-dir", "true").execute_azcopy_verify() + self.assertTrue(result) + + # sync to local + src = vdir_sas + dst = dir_path + result = util.Command("sync").add_arguments(src).add_arguments(dst).add_flags("log-level", "info")\ + .execute_azcopy_copy_command() + self.assertTrue(result) + + # sync back to blob after recreating the files + util.create_test_n_files(1024, 10, sub_dir_name) + src = dir_path + dst = vdir_sas + result = util.Command("sync").add_arguments(src).add_arguments(dst).add_flags("log-level", "info") \ + .execute_azcopy_copy_command() + self.assertTrue(result) From 28b16104d3417bdb3f5fd4c5b4aeee246ada5afe Mon Sep 17 00:00:00 2001 From: zezha-msft Date: Fri, 1 Mar 2019 13:18:47 -0800 Subject: [PATCH 64/64] 10.0.8 Release + changelog update --- ChangeLog.md | 12 ++++++++++-- cmd/copy.go | 2 +- cmd/sync.go | 2 +- common/version.go | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 0329bbfa4..7b26795b0 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,8 +1,16 @@ # Change Log -## Version X.XX.XX: +## Version 10.0.8: -- Command line parameter names changed as follows (to be consistent with naming pattern of other parameters) +- Rewrote sync command to eliminate numerous bugs and improve usability (see wiki for details) +- Implemented various improvements to memory management +- Added MD5 validation support (available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing) +- Added last modified time checks for source to guarantee transfer integrity +- Formalized outputs in JSON and elevated the output flag to the root level +- Eliminated outputs to STDERR (for new version notifications), which were causing problems for certain CI systems +- Improved log format for Windows +- Optimized plan file sizes +- Improved command line parameter names as follows (to be consistent with naming pattern of other parameters): - fromTo -> from-to - blobType -> blob-type - excludedBlobType -> excluded-blob-type diff --git a/cmd/copy.go b/cmd/copy.go index 59b27dd4a..5aa3a37d5 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -957,7 +957,7 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.contentEncoding, "content-encoding", "", "upload to Azure Storage using this content encoding.") cpCmd.PersistentFlags().BoolVar(&raw.noGuessMimeType, "no-guess-mime-type", false, "prevents AzCopy from detecting the content-type based on the extension/content of the file.") cpCmd.PersistentFlags().BoolVar(&raw.preserveLastModifiedTime, "preserve-last-modified-time", false, "only available when destination is file system.") - cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading.") + cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing.") // TODO: should the previous line list the allowable values? cpCmd.PersistentFlags().BoolVar(&raw.cancelFromStdin, "cancel-from-stdin", false, "true if user wants to cancel the process by passing 'cancel' "+ diff --git a/cmd/sync.go b/cmd/sync.go index 45f8895b3..681a11fcc 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -506,7 +506,7 @@ func init() { syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), and ERROR(only failed requests).") syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "false", "defines whether to delete extra files from the destination that are not present at the source. Could be set to true, false, or prompt. "+ "If set to prompt, user will be asked a question before scheduling files/blobs for deletion.") - syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading.") + syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "md5-validation", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing.") // TODO: should the previous line list the allowable values? // TODO follow sym link is not implemented, clarify behavior first diff --git a/common/version.go b/common/version.go index 330dd5259..3686e9fec 100644 --- a/common/version.go +++ b/common/version.go @@ -1,4 +1,4 @@ package common -const AzcopyVersion = "10.0.7-Preview" +const AzcopyVersion = "10.0.8-Preview" const UserAgent = "AzCopy/" + AzcopyVersion